HIVE-29322: Avoid TopNKeyOperator When Map-Side LIMIT Pushdown Provides Better Pruning

Author: Indhumathi27

iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out

@@ -66,8 +66,10 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
     ReduceSinkOperator reduceSinkOperator = (ReduceSinkOperator) nd;
     ReduceSinkDesc reduceSinkDesc = reduceSinkOperator.getConf();
 
-    // Check whether the reduce sink operator contains top n
-    if (reduceSinkDesc.getTopN() < 0 || !reduceSinkDesc.isOrdering()) {
+    // HIVE-29322: Skip creating TopNKeyOperator when LIMIT pushdown has already applied (topN >= -1)
+    // and the query uses a single reducer with no partition columns. In this scenario,
+    // TopNKey offers no extra pruning benefit and only adds unnecessary processing overhead.
+    if (reduceSinkDesc.getTopN() >= -1 || !reduceSinkDesc.isOrdering()) {
       return null;
     }
 

ql/src/java/org/apache/hadoop/hive/ql/optimizer/topnkey/TopNKeyProcessor.java

@@ -74,22 +74,17 @@ STAGE PLANS:
                   Filter Operator
                     predicate: cint is not null (type: boolean)
                     Statistics: Num rows: 9173 Data size: 671202 Basic stats: COMPLETE Column stats: COMPLETE
-                    Top N Key Operator
-                      sort order: +
-                      keys: cint (type: int)
-                      null sort order: z
-                      Statistics: Num rows: 9173 Data size: 671202 Basic stats: COMPLETE Column stats: COMPLETE
-                      top n: 10
-                      Select Operator
-                        expressions: cint (type: int), CAST( cstring1 AS varchar(128)) (type: varchar(128))
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: int)
-                          null sort order: z
-                          sort order: +
-                          Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE
-                          value expressions: _col1 (type: varchar(128))
+                    Select Operator
+                      expressions: cint (type: int), CAST( cstring1 AS varchar(128)) (type: varchar(128))
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: int)
+                        null sort order: z
+                        sort order: +
+                        Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE
+                        TopN Hash Memory Usage: 0.1
+                        value expressions: _col1 (type: varchar(128))
             Execution mode: vectorized, llap
             LLAP IO: all inputs
         Reducer 2 
@@ -98,27 +93,27 @@ STAGE PLANS:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(128))
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE
                 Limit
                   Number of rows: 10
-                  Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
                   Reduce Output Operator
                     key expressions: _col0 (type: int)
                     null sort order: a
                     sort order: +
                     Map-reduce partition columns: _col0 (type: int)
-                    Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
+                    Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
                     value expressions: _col1 (type: varchar(128))
         Reducer 3 
             Execution mode: vectorized, llap
             Reduce Operator Tree:
               Select Operator
                 expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(128))
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
+                Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
                 File Output Operator
                   compressed: false
-                  Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
                   table:
                       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
                       output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
@@ -128,7 +123,7 @@ STAGE PLANS:
                 Select Operator
                   expressions: _col0 (type: int), _col1 (type: varchar(128))
                   outputColumnNames: a, b
-                  Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
+                  Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
                   Group By Operator
                     aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), max(length(b)), avg(COALESCE(length(b),0)), count(b), compute_bit_vector_hll(b)
                     minReductionHashAggr: 0.9

ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out

@@ -75,17 +75,12 @@ STAGE PLANS:
                   1 _col0 (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE
-                Top N Key Operator
-                  sort order: ++
-                  keys: _col0 (type: string), _col1 (type: string)
+                Reduce Output Operator
+                  key expressions: _col0 (type: string), _col1 (type: string)
                   null sort order: zz
+                  sort order: ++
                   Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE
-                  top n: 40
-                  Reduce Output Operator
-                    key expressions: _col0 (type: string), _col1 (type: string)
-                    null sort order: zz
-                    sort order: ++
-                    Statistics: Num rows: 791 Data size: 140798 Basic stats: COMPLETE Column stats: COMPLETE
+                  TopN Hash Memory Usage: 0.1
         Reducer 3 
             Execution mode: llap
             Reduce Operator Tree:
@@ -268,17 +263,12 @@ STAGE PLANS:
                   1 _col0 (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 1288 Data size: 229264 Basic stats: COMPLETE Column stats: COMPLETE
-                Top N Key Operator
-                  sort order: ++
-                  keys: _col0 (type: string), _col1 (type: string)
+                Reduce Output Operator
+                  key expressions: _col0 (type: string), _col1 (type: string)
                   null sort order: zz
+                  sort order: ++
                   Statistics: Num rows: 1288 Data size: 229264 Basic stats: COMPLETE Column stats: COMPLETE
-                  top n: 40
-                  Reduce Output Operator
-                    key expressions: _col0 (type: string), _col1 (type: string)
-                    null sort order: zz
-                    sort order: ++
-                    Statistics: Num rows: 1288 Data size: 229264 Basic stats: COMPLETE Column stats: COMPLETE
+                  TopN Hash Memory Usage: 0.1
         Reducer 4 
             Execution mode: llap
             Reduce Operator Tree:
@@ -461,17 +451,12 @@ STAGE PLANS:
                   1 _col0 (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 270 Data size: 48060 Basic stats: COMPLETE Column stats: COMPLETE
-                Top N Key Operator
-                  sort order: ++
-                  keys: _col0 (type: string), _col1 (type: string)
+                Reduce Output Operator
+                  key expressions: _col0 (type: string), _col1 (type: string)
                   null sort order: zz
+                  sort order: ++
                   Statistics: Num rows: 270 Data size: 48060 Basic stats: COMPLETE Column stats: COMPLETE
-                  top n: 40
-                  Reduce Output Operator
-                    key expressions: _col0 (type: string), _col1 (type: string)
-                    null sort order: zz
-                    sort order: ++
-                    Statistics: Num rows: 270 Data size: 48060 Basic stats: COMPLETE Column stats: COMPLETE
+                  TopN Hash Memory Usage: 0.1
         Reducer 4 
             Execution mode: llap
             Reduce Operator Tree: