Skip to content

Commit e8261fb

Browse files
andygroveandygrove
authored
fix: Fall back to Spark for distinct aggregates (#1262)
* fall back to Spark for distinct aggregates * update expected plans for 3.4 * update expected plans for 3.5 * force build * add comment
1 parent c25060e commit e8261fb

File tree

18 files changed

+207
-163
lines changed

18 files changed

+207
-163
lines changed

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,13 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
371371
inputs: Seq[Attribute],
372372
binding: Boolean,
373373
conf: SQLConf): Option[AggExpr] = {
374+
375+
if (aggExpr.isDistinct) {
376+
// https://github.com/apache/datafusion-comet/issues/1260
377+
withInfo(aggExpr, "distinct aggregates are not supported")
378+
return None
379+
}
380+
374381
aggExpr.aggregateFunction match {
375382
case s @ Sum(child, _) if sumDataTypeSupported(s.dataType) && isLegacyMode(s) =>
376383
val childExpr = exprToProto(child, inputs, binding)

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
== Physical Plan ==
2-
* CometColumnarToRow (41)
3-
+- CometHashAggregate (40)
2+
* HashAggregate (41)
3+
+- * CometColumnarToRow (40)
44
+- CometColumnarExchange (39)
55
+- * HashAggregate (38)
66
+- * HashAggregate (37)
@@ -225,11 +225,13 @@ Results [3]: [sum#20, sum#21, count#25]
225225
Input [3]: [sum#20, sum#21, count#25]
226226
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4]
227227

228-
(40) CometHashAggregate
228+
(40) CometColumnarToRow [codegen id : 2]
229+
Input [3]: [sum#20, sum#21, count#25]
230+
231+
(41) HashAggregate [codegen id : 2]
229232
Input [3]: [sum#20, sum#21, count#25]
230233
Keys: []
231234
Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)]
232-
233-
(41) CometColumnarToRow [codegen id : 2]
234-
Input [3]: [order count #26, total shipping cost #27, total net profit #28]
235+
Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#22, sum(UnscaledValue(cs_net_profit#7))#23, count(cs_order_number#5)#24]
236+
Results [3]: [count(cs_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(cs_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(cs_net_profit#7))#23,17,2) AS total net profit #28]
235237

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
WholeStageCodegen (2)
2-
CometColumnarToRow
3-
InputAdapter
4-
CometHashAggregate [order count ,total shipping cost ,total net profit ,sum,sum,count,count(cs_order_number),sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit))]
2+
HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
3+
CometColumnarToRow
4+
InputAdapter
55
CometColumnarExchange #1
66
WholeStageCodegen (1)
77
HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count]

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/explain.txt‎

Lines changed: 48 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
: :- * BroadcastNestedLoopJoin Inner BuildRight (50)
55
: : :- * BroadcastNestedLoopJoin Inner BuildRight (37)
66
: : : :- * BroadcastNestedLoopJoin Inner BuildRight (24)
7-
: : : : :- * CometColumnarToRow (11)
8-
: : : : : +- CometHashAggregate (10)
7+
: : : : :- * HashAggregate (11)
8+
: : : : : +- * CometColumnarToRow (10)
99
: : : : : +- CometColumnarExchange (9)
1010
: : : : : +- * HashAggregate (8)
1111
: : : : : +- * HashAggregate (7)
@@ -16,8 +16,8 @@
1616
: : : : : +- CometFilter (2)
1717
: : : : : +- CometScan parquet spark_catalog.default.store_sales (1)
1818
: : : : +- BroadcastExchange (23)
19-
: : : : +- * CometColumnarToRow (22)
20-
: : : : +- CometHashAggregate (21)
19+
: : : : +- * HashAggregate (22)
20+
: : : : +- * CometColumnarToRow (21)
2121
: : : : +- CometColumnarExchange (20)
2222
: : : : +- * HashAggregate (19)
2323
: : : : +- * HashAggregate (18)
@@ -28,8 +28,8 @@
2828
: : : : +- CometFilter (13)
2929
: : : : +- CometScan parquet spark_catalog.default.store_sales (12)
3030
: : : +- BroadcastExchange (36)
31-
: : : +- * CometColumnarToRow (35)
32-
: : : +- CometHashAggregate (34)
31+
: : : +- * HashAggregate (35)
32+
: : : +- * CometColumnarToRow (34)
3333
: : : +- CometColumnarExchange (33)
3434
: : : +- * HashAggregate (32)
3535
: : : +- * HashAggregate (31)
@@ -40,8 +40,8 @@
4040
: : : +- CometFilter (26)
4141
: : : +- CometScan parquet spark_catalog.default.store_sales (25)
4242
: : +- BroadcastExchange (49)
43-
: : +- * CometColumnarToRow (48)
44-
: : +- CometHashAggregate (47)
43+
: : +- * HashAggregate (48)
44+
: : +- * CometColumnarToRow (47)
4545
: : +- CometColumnarExchange (46)
4646
: : +- * HashAggregate (45)
4747
: : +- * HashAggregate (44)
@@ -52,8 +52,8 @@
5252
: : +- CometFilter (39)
5353
: : +- CometScan parquet spark_catalog.default.store_sales (38)
5454
: +- BroadcastExchange (62)
55-
: +- * CometColumnarToRow (61)
56-
: +- CometHashAggregate (60)
55+
: +- * HashAggregate (61)
56+
: +- * CometColumnarToRow (60)
5757
: +- CometColumnarExchange (59)
5858
: +- * HashAggregate (58)
5959
: +- * HashAggregate (57)
@@ -64,8 +64,8 @@
6464
: +- CometFilter (52)
6565
: +- CometScan parquet spark_catalog.default.store_sales (51)
6666
+- BroadcastExchange (75)
67-
+- * CometColumnarToRow (74)
68-
+- CometHashAggregate (73)
67+
+- * HashAggregate (74)
68+
+- * CometColumnarToRow (73)
6969
+- CometColumnarExchange (72)
7070
+- * HashAggregate (71)
7171
+- * HashAggregate (70)
@@ -122,13 +122,15 @@ Results [4]: [sum#6, count#7, count#8, count#12]
122122
Input [4]: [sum#6, count#7, count#8, count#12]
123123
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2]
124124

125-
(10) CometHashAggregate
125+
(10) CometColumnarToRow [codegen id : 12]
126+
Input [4]: [sum#6, count#7, count#8, count#12]
127+
128+
(11) HashAggregate [codegen id : 12]
126129
Input [4]: [sum#6, count#7, count#8, count#12]
127130
Keys: []
128131
Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)]
129-
130-
(11) CometColumnarToRow [codegen id : 12]
131-
Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15]
132+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11]
133+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15]
132134

133135
(12) CometScan parquet spark_catalog.default.store_sales
134136
Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20]
@@ -175,13 +177,15 @@ Results [4]: [sum#21, count#22, count#23, count#27]
175177
Input [4]: [sum#21, count#22, count#23, count#27]
176178
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4]
177179

178-
(21) CometHashAggregate
180+
(21) CometColumnarToRow [codegen id : 3]
181+
Input [4]: [sum#21, count#22, count#23, count#27]
182+
183+
(22) HashAggregate [codegen id : 3]
179184
Input [4]: [sum#21, count#22, count#23, count#27]
180185
Keys: []
181186
Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)]
182-
183-
(22) CometColumnarToRow [codegen id : 3]
184-
Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30]
187+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26]
188+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30]
185189

186190
(23) BroadcastExchange
187191
Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30]
@@ -236,13 +240,15 @@ Results [4]: [sum#36, count#37, count#38, count#42]
236240
Input [4]: [sum#36, count#37, count#38, count#42]
237241
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7]
238242

239-
(34) CometHashAggregate
243+
(34) CometColumnarToRow [codegen id : 5]
244+
Input [4]: [sum#36, count#37, count#38, count#42]
245+
246+
(35) HashAggregate [codegen id : 5]
240247
Input [4]: [sum#36, count#37, count#38, count#42]
241248
Keys: []
242249
Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)]
243-
244-
(35) CometColumnarToRow [codegen id : 5]
245-
Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45]
250+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41]
251+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45]
246252

247253
(36) BroadcastExchange
248254
Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45]
@@ -297,13 +303,15 @@ Results [4]: [sum#51, count#52, count#53, count#57]
297303
Input [4]: [sum#51, count#52, count#53, count#57]
298304
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10]
299305

300-
(47) CometHashAggregate
306+
(47) CometColumnarToRow [codegen id : 7]
307+
Input [4]: [sum#51, count#52, count#53, count#57]
308+
309+
(48) HashAggregate [codegen id : 7]
301310
Input [4]: [sum#51, count#52, count#53, count#57]
302311
Keys: []
303312
Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)]
304-
305-
(48) CometColumnarToRow [codegen id : 7]
306-
Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60]
313+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56]
314+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60]
307315

308316
(49) BroadcastExchange
309317
Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60]
@@ -358,13 +366,15 @@ Results [4]: [sum#66, count#67, count#68, count#72]
358366
Input [4]: [sum#66, count#67, count#68, count#72]
359367
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13]
360368

361-
(60) CometHashAggregate
369+
(60) CometColumnarToRow [codegen id : 9]
370+
Input [4]: [sum#66, count#67, count#68, count#72]
371+
372+
(61) HashAggregate [codegen id : 9]
362373
Input [4]: [sum#66, count#67, count#68, count#72]
363374
Keys: []
364375
Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)]
365-
366-
(61) CometColumnarToRow [codegen id : 9]
367-
Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75]
376+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71]
377+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75]
368378

369379
(62) BroadcastExchange
370380
Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75]
@@ -419,13 +429,15 @@ Results [4]: [sum#81, count#82, count#83, count#87]
419429
Input [4]: [sum#81, count#82, count#83, count#87]
420430
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16]
421431

422-
(73) CometHashAggregate
432+
(73) CometColumnarToRow [codegen id : 11]
433+
Input [4]: [sum#81, count#82, count#83, count#87]
434+
435+
(74) HashAggregate [codegen id : 11]
423436
Input [4]: [sum#81, count#82, count#83, count#87]
424437
Keys: []
425438
Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)]
426-
427-
(74) CometColumnarToRow [codegen id : 11]
428-
Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90]
439+
Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86]
440+
Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90]
429441

430442
(75) BroadcastExchange
431443
Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90]

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/simplified.txt‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ WholeStageCodegen (12)
44
BroadcastNestedLoopJoin
55
BroadcastNestedLoopJoin
66
BroadcastNestedLoopJoin
7-
CometColumnarToRow
8-
InputAdapter
9-
CometHashAggregate [B1_LP,B1_CNT,B1_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
7+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count]
8+
CometColumnarToRow
9+
InputAdapter
1010
CometColumnarExchange #1
1111
WholeStageCodegen (1)
1212
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
@@ -21,9 +21,9 @@ WholeStageCodegen (12)
2121
InputAdapter
2222
BroadcastExchange #3
2323
WholeStageCodegen (3)
24-
CometColumnarToRow
25-
InputAdapter
26-
CometHashAggregate [B2_LP,B2_CNT,B2_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
24+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count]
25+
CometColumnarToRow
26+
InputAdapter
2727
CometColumnarExchange #4
2828
WholeStageCodegen (2)
2929
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
@@ -38,9 +38,9 @@ WholeStageCodegen (12)
3838
InputAdapter
3939
BroadcastExchange #6
4040
WholeStageCodegen (5)
41-
CometColumnarToRow
42-
InputAdapter
43-
CometHashAggregate [B3_LP,B3_CNT,B3_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
41+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count]
42+
CometColumnarToRow
43+
InputAdapter
4444
CometColumnarExchange #7
4545
WholeStageCodegen (4)
4646
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
@@ -55,9 +55,9 @@ WholeStageCodegen (12)
5555
InputAdapter
5656
BroadcastExchange #9
5757
WholeStageCodegen (7)
58-
CometColumnarToRow
59-
InputAdapter
60-
CometHashAggregate [B4_LP,B4_CNT,B4_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
58+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count]
59+
CometColumnarToRow
60+
InputAdapter
6161
CometColumnarExchange #10
6262
WholeStageCodegen (6)
6363
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
@@ -72,9 +72,9 @@ WholeStageCodegen (12)
7272
InputAdapter
7373
BroadcastExchange #12
7474
WholeStageCodegen (9)
75-
CometColumnarToRow
76-
InputAdapter
77-
CometHashAggregate [B5_LP,B5_CNT,B5_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
75+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count]
76+
CometColumnarToRow
77+
InputAdapter
7878
CometColumnarExchange #13
7979
WholeStageCodegen (8)
8080
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
@@ -89,9 +89,9 @@ WholeStageCodegen (12)
8989
InputAdapter
9090
BroadcastExchange #15
9191
WholeStageCodegen (11)
92-
CometColumnarToRow
93-
InputAdapter
94-
CometHashAggregate [B6_LP,B6_CNT,B6_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)]
92+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count]
93+
CometColumnarToRow
94+
InputAdapter
9595
CometColumnarExchange #16
9696
WholeStageCodegen (10)
9797
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/explain.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
== Physical Plan ==
2-
* CometColumnarToRow (41)
3-
+- CometHashAggregate (40)
2+
* HashAggregate (41)
3+
+- * CometColumnarToRow (40)
44
+- CometColumnarExchange (39)
55
+- * HashAggregate (38)
66
+- * HashAggregate (37)
@@ -225,11 +225,13 @@ Results [3]: [sum#20, sum#21, count#25]
225225
Input [3]: [sum#20, sum#21, count#25]
226226
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4]
227227

228-
(40) CometHashAggregate
228+
(40) CometColumnarToRow [codegen id : 2]
229+
Input [3]: [sum#20, sum#21, count#25]
230+
231+
(41) HashAggregate [codegen id : 2]
229232
Input [3]: [sum#20, sum#21, count#25]
230233
Keys: []
231234
Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)]
232-
233-
(41) CometColumnarToRow [codegen id : 2]
234-
Input [3]: [order count #26, total shipping cost #27, total net profit #28]
235+
Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#22, sum(UnscaledValue(ws_net_profit#7))#23, count(ws_order_number#5)#24]
236+
Results [3]: [count(ws_order_number#5)#24 AS order count #26, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#6))#22,17,2) AS total shipping cost #27, MakeDecimal(sum(UnscaledValue(ws_net_profit#7))#23,17,2) AS total net profit #28]
235237

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
WholeStageCodegen (2)
2-
CometColumnarToRow
3-
InputAdapter
4-
CometHashAggregate [order count ,total shipping cost ,total net profit ,sum,sum,count,count(ws_order_number),sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit))]
2+
HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count]
3+
CometColumnarToRow
4+
InputAdapter
55
CometColumnarExchange #1
66
WholeStageCodegen (1)
77
HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count]

‎spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
== Physical Plan ==
2-
* CometColumnarToRow (54)
3-
+- CometHashAggregate (53)
2+
* HashAggregate (54)
3+
+- * CometColumnarToRow (53)
44
+- CometColumnarExchange (52)
55
+- * HashAggregate (51)
66
+- * HashAggregate (50)
@@ -292,11 +292,13 @@ Results [3]: [sum#25, sum#26, count#30]
292292
Input [3]: [sum#25, sum#26, count#30]
293293
Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4]
294294

295-
(53) CometHashAggregate
295+
(53) CometColumnarToRow [codegen id : 2]
296+
Input [3]: [sum#25, sum#26, count#30]
297+
298+
(54) HashAggregate [codegen id : 2]
296299
Input [3]: [sum#25, sum#26, count#30]
297300
Keys: []
298301
Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)]
299-
300-
(54) CometColumnarToRow [codegen id : 2]
301-
Input [3]: [order count #31, total shipping cost #32, total net profit #33]
302+
Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#27, sum(UnscaledValue(ws_net_profit#6))#28, count(ws_order_number#4)#29]
303+
Results [3]: [count(ws_order_number#4)#29 AS order count #31, MakeDecimal(sum(UnscaledValue(ws_ext_ship_cost#5))#27,17,2) AS total shipping cost #32, MakeDecimal(sum(UnscaledValue(ws_net_profit#6))#28,17,2) AS total net profit #33]
302304

0 commit comments

Comments
 (0)