diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 2a947c5e0eed..1de11eb81f2a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -54,9 +54,11 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; +import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; +import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -1300,9 +1302,10 @@ private static void runTopNKeyOptimization(OptimizeTezProcContext procCtx) return; } + String topNKeyRegexPattern = buildTopNKeyRegexPattern(procCtx); Map opRules = new LinkedHashMap(); opRules.put( - new RuleRegExp("Top n key optimization", ReduceSinkOperator.getOperatorName() + "%"), + new RuleRegExp("Top n key optimization", topNKeyRegexPattern), new TopNKeyProcessor( HiveConf.getIntVar(procCtx.conf, HiveConf.ConfVars.HIVE_MAX_TOPN_ALLOWED), HiveConf.getFloatVar(procCtx.conf, ConfVars.HIVE_TOPN_EFFICIENCY_THRESHOLD), @@ -1322,6 +1325,49 @@ private static void runTopNKeyOptimization(OptimizeTezProcContext procCtx) ogw.startWalking(topNodes, null); } + /* + * Build the ReduceSink matching pattern used by TopNKey optimization. + * + * For ORDER BY / LIMIT queries that do not involve GROUP BY or JOIN, + * applying TopNKey results in a performance regression. ReduceSink + * operators created only for ordering must therefore be excluded from + * TopNKey. + * + * When ORDER BY or LIMIT is present, restrict TopNKey to ReduceSink + * operators that originate from GROUP BY, JOIN, MAPJOIN, LATERAL VIEW + * JOIN or PTF query shapes + */ + private static String buildTopNKeyRegexPattern(OptimizeTezProcContext procCtx) { + String reduceSinkOp = ReduceSinkOperator.getOperatorName() + "%"; + + boolean hasOrderOrLimit = + procCtx.parseContext.getQueryProperties().hasLimit() || + procCtx.parseContext.getQueryProperties().hasOrderBy(); + + if (hasPTFReduceSink(procCtx) || !hasOrderOrLimit) { + return reduceSinkOp; + } + + return "(" + + GroupByOperator.getOperatorName() + "|" + + PTFOperator.getOperatorName() + "|" + + JoinOperator.getOperatorName() + "|" + + MapJoinOperator.getOperatorName() + "|" + + LateralViewJoinOperator.getOperatorName() + "|" + + CommonMergeJoinOperator.getOperatorName() + + ").*%" + + reduceSinkOp; + } + + private static boolean hasPTFReduceSink(OptimizeTezProcContext ctx) { + for (ReduceSinkOperator rs : ctx.visitedReduceSinks) { + if (rs.getConf().isPTFReduceSink()) { + return true; + } + } + return false; + } + private boolean findParallelSemiJoinBranch(Operator mapjoin, TableScanOperator bigTableTS, ParseContext parseContext, Map semijoins, diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out index 94fd96f7ee19..b4cc3fca8798 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out @@ -74,22 +74,17 @@ STAGE PLANS: Filter Operator predicate: cint is not null (type: boolean) Statistics: Num rows: 9173 Data size: 671202 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: cint (type: int) - null sort order: z - Statistics: Num rows: 9173 Data size: 671202 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: cint (type: int), CAST( cstring1 AS varchar(128)) (type: varchar(128)) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: varchar(128)) + Select Operator + expressions: cint (type: int), CAST( cstring1 AS varchar(128)) (type: varchar(128)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: varchar(128)) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -98,16 +93,16 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(128)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 9173 Data size: 977184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9173 Data size: 1479384 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: varchar(128)) Reducer 3 Execution mode: vectorized, llap @@ -115,10 +110,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(128)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -128,7 +123,7 @@ STAGE PLANS: Select Operator expressions: _col0 (type: int), _col1 (type: varchar(128)) outputColumnNames: a, b - Statistics: Num rows: 10 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(a), max(a), count(1), count(a), compute_bit_vector_hll(a), max(length(b)), avg(COALESCE(length(b),0)), count(b), compute_bit_vector_hll(b) minReductionHashAggr: 0.9 diff --git a/ql/src/test/results/clientpositive/llap/cbo_SortUnionTransposeRule.q.out b/ql/src/test/results/clientpositive/llap/cbo_SortUnionTransposeRule.q.out index 1cbfc2f1e176..36c75c1f018b 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_SortUnionTransposeRule.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_SortUnionTransposeRule.q.out @@ -242,17 +242,12 @@ STAGE PLANS: expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -264,17 +259,12 @@ STAGE PLANS: expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 3 @@ -719,26 +709,22 @@ STAGE PLANS: TableScan alias: a Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -751,17 +737,12 @@ STAGE PLANS: Limit Number of rows: 5 Statistics: Num rows: 5 Data size: 435 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 4 Execution mode: vectorized, llap Reduce Operator Tree: @@ -789,17 +770,12 @@ STAGE PLANS: Limit Number of rows: 5 Statistics: Num rows: 5 Data size: 435 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 870 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Union 3 Vertex: Union 3 diff --git a/ql/src/test/results/clientpositive/llap/cbo_input26.q.out b/ql/src/test/results/clientpositive/llap/cbo_input26.q.out index 722b72df0f90..564f46c4aa98 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_input26.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_input26.q.out @@ -37,22 +37,17 @@ STAGE PLANS: alias: a filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -196,21 +191,16 @@ STAGE PLANS: alias: a filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -354,21 +344,16 @@ STAGE PLANS: alias: a filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map 4 diff --git a/ql/src/test/results/clientpositive/llap/check_constraint.q.out b/ql/src/test/results/clientpositive/llap/check_constraint.q.out index 43c9a641a21a..bcf964864cbd 100644 --- a/ql/src/test/results/clientpositive/llap/check_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/check_constraint.q.out @@ -1461,22 +1461,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: UDFToInteger(key) (type: int) - null sort order: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -3769,22 +3764,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: value (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/constraints_explain_ddl.q.out b/ql/src/test/results/clientpositive/llap/constraints_explain_ddl.q.out index 98c8a9056d22..41585685c24f 100644 --- a/ql/src/test/results/clientpositive/llap/constraints_explain_ddl.q.out +++ b/ql/src/test/results/clientpositive/llap/constraints_explain_ddl.q.out @@ -383,21 +383,16 @@ STAGE PLANS: Filter Operator predicate: (d_year) IN (1985, 2004) (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: d_datekey (type: bigint) - null sort order: z + Select Operator + expressions: d_datekey (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: d_datekey (type: bigint) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -529,21 +524,16 @@ STAGE PLANS: Filter Operator predicate: (d_year) IN (1985, 2004) (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: d_datekey (type: bigint) - null sort order: z + Select Operator + expressions: d_datekey (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: d_datekey (type: bigint) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/constraints_optimization.q.out b/ql/src/test/results/clientpositive/llap/constraints_optimization.q.out index 1e537e875532..8b23e711f756 100644 --- a/ql/src/test/results/clientpositive/llap/constraints_optimization.q.out +++ b/ql/src/test/results/clientpositive/llap/constraints_optimization.q.out @@ -187,21 +187,16 @@ STAGE PLANS: Filter Operator predicate: (d_year) IN (1985, 2004) (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: d_datekey (type: bigint) - null sort order: z + Select Operator + expressions: d_datekey (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: d_datekey (type: bigint) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -261,21 +256,16 @@ STAGE PLANS: Filter Operator predicate: (d_year) IN (1985, 2004) (type: boolean) Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: d_datekey (type: bigint) - null sort order: z + Select Operator + expressions: d_datekey (type: bigint) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: d_datekey (type: bigint) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: bigint) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/cp_sel.q.out b/ql/src/test/results/clientpositive/llap/cp_sel.q.out index 590fc61999c8..166047f3f9c7 100644 --- a/ql/src/test/results/clientpositive/llap/cp_sel.q.out +++ b/ql/src/test/results/clientpositive/llap/cp_sel.q.out @@ -30,22 +30,17 @@ STAGE PLANS: alias: srcpart filterExpr: (11.0D = 11.0D) (type: boolean) Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 1 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out b/ql/src/test/results/clientpositive/llap/ctas.q.out index 88fb86dcf767..a2261c5ca2ff 100644 --- a/ql/src/test/results/clientpositive/llap/ctas.q.out +++ b/ql/src/test/results/clientpositive/llap/ctas.q.out @@ -46,21 +46,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -69,21 +64,16 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - null sort order: zz + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -259,21 +249,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -282,21 +267,16 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - null sort order: zz + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -472,21 +452,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: (key / 2) (type: double), concat(value, '_con') (type: string) - null sort order: zz - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: (key / 2) (type: double), concat(value, '_con') (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: (key / 2) (type: double), concat(value, '_con') (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 96000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -495,21 +470,16 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string) - null sort order: zz + Select Operator + expressions: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 10 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -749,21 +719,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -772,21 +737,16 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - null sort order: zz + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: @@ -964,21 +924,16 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -987,21 +942,16 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - null sort order: zz + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/decimal_stats.q.out b/ql/src/test/results/clientpositive/llap/decimal_stats.q.out index efaa4f785522..86d43defb0c6 100644 --- a/ql/src/test/results/clientpositive/llap/decimal_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/decimal_stats.q.out @@ -88,22 +88,17 @@ STAGE PLANS: TableScan alias: decimal_1_n1 Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: t (type: decimal(4,2)) - null sort order: z + Select Operator + expressions: t (type: decimal(4,2)), u (type: decimal(5,0)), v (type: decimal(10,0)) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: t (type: decimal(4,2)), u (type: decimal(5,0)), v (type: decimal(10,0)) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 56112 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: decimal(4,2)) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 56112 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,0)), _col2 (type: decimal(10,0)) + Reduce Output Operator + key expressions: _col0 (type: decimal(4,2)) + null sort order: z + sort order: + + Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(5,0)), _col2 (type: decimal(10,0)) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -112,7 +107,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: decimal(4,2)), VALUE._col0 (type: decimal(5,0)), VALUE._col1 (type: decimal(10,0)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 56112 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 Statistics: Num rows: 100 Data size: 22512 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index cdccef6aa366..3070f60b3ce2 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -2006,22 +2006,17 @@ STAGE PLANS: TableScan alias: over1k_orc Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: i (type: int) - null sort order: z + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1049 Data size: 25136 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 1049 Data size: 25136 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + Reduce Output Operator + key expressions: _col2 (type: int) + null sort order: z + sort order: + + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -2030,7 +2025,7 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1049 Data size: 25136 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index aaca4368d0fd..53cbab6fb991 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -1935,22 +1935,17 @@ STAGE PLANS: TableScan alias: over1k_n3 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: i (type: int) - null sort order: z + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Reduce Output Operator + key expressions: _col2 (type: int) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out index 2d9d4f6070ad..ff6f35846bc3 100644 --- a/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out +++ b/ql/src/test/results/clientpositive/llap/enforce_constraint_notnull.q.out @@ -2994,22 +2994,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: UDFToInteger(key) (type: int) - null sort order: z - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 2 - Select Operator - expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) - outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: UDFToInteger(key) (type: int), CAST( key AS decimal(5,2)) (type: decimal(5,2)), value (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 103500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(5,2)), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -6287,22 +6282,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: value (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 7151b95c136f..48d57927d1d0 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -1574,34 +1574,28 @@ Stage-0 Stage-1 Reducer 4 llap File Output Operator [FS_13] - Limit [LIM_12] (rows=5 width=35) + Limit [LIM_12] (rows=5 width=89) Number of rows:5 - Select Operator [SEL_11] (rows=5 width=35) + Select Operator [SEL_11] (rows=5 width=89) Output:["_col0","_col1"] <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_10] - Top N Key Operator [TNK_14] (rows=5 width=71) - keys:_col1,top n:5 - Limit [LIM_8] (rows=5 width=71) - Number of rows:5 - Select Operator [SEL_7] (rows=5 width=71) - Output:["_col0","_col1"] - <-Reducer 2 [SIMPLE_EDGE] llap - SHUFFLE [RS_6] - Top N Key Operator [TNK_15] (rows=5 width=89) - keys:_col1,top n:5 - Limit [LIM_4] (rows=5 width=89) - Number of rows:5 - Select Operator [SEL_3] (rows=20 width=75) + Limit [LIM_8] (rows=5 width=89) + Number of rows:5 + Select Operator [SEL_7] (rows=5 width=89) + Output:["_col0","_col1"] + <-Reducer 2 [SIMPLE_EDGE] llap + SHUFFLE [RS_6] + Limit [LIM_4] (rows=5 width=89) + Number of rows:5 + Select Operator [SEL_3] (rows=20 width=84) + Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=20 width=84) Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=20 width=75) - Output:["_col0","_col1"] - Top N Key Operator [TNK_16] (rows=20 width=84) - keys:c_int,top n:5 - TableScan [TS_0] (rows=20 width=84) - default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] + TableScan [TS_0] (rows=20 width=84) + default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] PREHOOK: query: explain select cbo_t3.c_int, c, count(*) from (select key as a, c_int+1 as b, sum(c_int) as c from cbo_t1 where (cbo_t1.c_int + 1 >= 0) and (cbo_t1.c_int > 0 or cbo_t1.c_float >= 0) group by c_float, cbo_t1.c_int, key order by a limit 5) cbo_t1 join (select key as p, c_int+1 as q, sum(c_int) as r from cbo_t2 where (cbo_t2.c_int + 1 >= 0) and (cbo_t2.c_int > 0 or cbo_t2.c_float >= 0) group by c_float, cbo_t2.c_int, key order by q/10 desc, r asc limit 5) cbo_t2 on cbo_t1.a=p join cbo_t3 on cbo_t1.a=key where (b + cbo_t2.q >= 0) and (b > 0 or c_int >= 0) group by cbo_t3.c_int, c order by cbo_t3.c_int+c desc, c limit 5 PREHOOK: type: QUERY @@ -3725,18 +3719,14 @@ Stage-3 SHUFFLE [RS_5] Select Operator [SEL_3] (rows=10 width=178) Output:["_col0","_col1"] - Top N Key Operator [TNK_18] (rows=10 width=178) - keys:KEY.reducesinkkey0, KEY.reducesinkkey1,top n:10 - Limit [LIM_4] (rows=10 width=178) - Number of rows:10 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=178) - Output:["_col0","_col1"] - Top N Key Operator [TNK_19] (rows=500 width=178) - keys:key, value,top n:10 - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Limit [LIM_4] (rows=10 width=178) + Number of rows:10 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500 width=178) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_14] Select Operator [SEL_13] (rows=10 width=178) Output:["col1","col2"] @@ -3798,18 +3788,14 @@ Stage-3 SHUFFLE [RS_5] Select Operator [SEL_3] (rows=10 width=192) Output:["_col0","_col1"] - Top N Key Operator [TNK_18] (rows=10 width=192) - keys:KEY.reducesinkkey0, KEY.reducesinkkey1,top n:10 - Limit [LIM_4] (rows=10 width=192) - Number of rows:10 - <-Map 1 [SIMPLE_EDGE] llap - SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500 width=192) - Output:["_col0","_col1"] - Top N Key Operator [TNK_19] (rows=500 width=178) - keys:(key / 2), concat(value, '_con'),top n:10 - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Limit [LIM_4] (rows=10 width=192) + Number of rows:10 + <-Map 1 [SIMPLE_EDGE] llap + SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500 width=192) + Output:["_col0","_col1"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PARTITION_ONLY_SHUFFLE [RS_14] Select Operator [SEL_13] (rows=10 width=192) Output:["col1","col2"] diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out index a06b1eb40f82..e77483f13305 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table_perf.q.out @@ -6088,21 +6088,16 @@ GROUP BY "t4"."r_reason_desc" hive.sql.query.fieldTypes char(100),bigint,bigint,decimal(17,2),bigint,decimal(17,2),bigint hive.sql.query.split false Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: ++++ - keys: substr(r_reason_desc, 1, 20) (type: string), (UDFToDouble($f1) / $f2) (type: double), CAST( ($f3 / $f4) AS decimal(11,6)) (type: decimal(11,6)), CAST( ($f5 / $f6) AS decimal(11,6)) (type: decimal(11,6)) - null sort order: zzzz + Select Operator + expressions: substr(r_reason_desc, 1, 20) (type: string), (UDFToDouble($f1) / $f2) (type: double), CAST( ($f3 / $f4) AS decimal(11,6)) (type: decimal(11,6)), CAST( ($f5 / $f6) AS decimal(11,6)) (type: decimal(11,6)) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE - top n: 100 - Select Operator - expressions: substr(r_reason_desc, 1, 20) (type: string), (UDFToDouble($f1) / $f2) (type: double), CAST( ($f3 / $f4) AS decimal(11,6)) (type: decimal(11,6)), CAST( ($f5 / $f6) AS decimal(11,6)) (type: decimal(11,6)) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: double), _col2 (type: decimal(11,6)), _col3 (type: decimal(11,6)) + null sort order: zzzz + sort order: ++++ Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: double), _col2 (type: decimal(11,6)), _col3 (type: decimal(11,6)) - null sort order: zzzz - sort order: ++++ - Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out b/ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out index 9631a1792f1a..2d4c321b6886 100644 --- a/ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out +++ b/ql/src/test/results/clientpositive/llap/gen_udf_example_add10.q.out @@ -45,21 +45,16 @@ STAGE PLANS: TableScan alias: t1_n102 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: -+ - keys: add10(x) (type: int), add10(y) (type: double) - null sort order: az + Select Operator + expressions: add10(x) (type: int), add10(y) (type: double) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: add10(x) (type: int), add10(y) (type: double) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double) + null sort order: az + sort order: -+ Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: double) - null sort order: az - sort order: -+ - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized Reducer 2 Execution mode: vectorized, llap diff --git a/ql/src/test/results/clientpositive/llap/input14_limit.q.out b/ql/src/test/results/clientpositive/llap/input14_limit.q.out index 070a4019768e..4296a27d6cc7 100644 --- a/ql/src/test/results/clientpositive/llap/input14_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/input14_limit.q.out @@ -59,42 +59,32 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string) Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Limit Number of rows: 20 Statistics: Num rows: 20 Data size: 5300 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: VALUE._col0 (type: string) - null sort order: a - Statistics: Num rows: 20 Data size: 5300 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/input22.q.out b/ql/src/test/results/clientpositive/llap/input22.q.out index 48bf832c3a6f..f6f891d0304e 100644 --- a/ql/src/test/results/clientpositive/llap/input22.q.out +++ b/ql/src/test/results/clientpositive/llap/input22.q.out @@ -47,21 +47,16 @@ STAGE PLANS: TableScan alias: input4 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: key (type: string) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/input26.q.out b/ql/src/test/results/clientpositive/llap/input26.q.out index 507d39e66ae7..2d45544c0e54 100644 --- a/ql/src/test/results/clientpositive/llap/input26.q.out +++ b/ql/src/test/results/clientpositive/llap/input26.q.out @@ -37,22 +37,17 @@ STAGE PLANS: alias: a filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 diff --git a/ql/src/test/results/clientpositive/llap/input3_limit.q.out b/ql/src/test/results/clientpositive/llap/input3_limit.q.out index bc3f74f6a183..4d8a3bb814c0 100644 --- a/ql/src/test/results/clientpositive/llap/input3_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/input3_limit.q.out @@ -76,21 +76,16 @@ STAGE PLANS: Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: - Top N Key Operator - sort order: ++ - keys: VALUE._col0 (type: string), VALUE._col1 (type: string) - null sort order: zz + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - top n: 20 - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/input4_limit.q.out b/ql/src/test/results/clientpositive/llap/input4_limit.q.out index a64dca1f3641..a647bc9c21f8 100644 --- a/ql/src/test/results/clientpositive/llap/input4_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/input4_limit.q.out @@ -26,22 +26,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -50,22 +45,17 @@ STAGE PLANS: Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: KEY.reducesinkkey0 (type: string) - null sort order: z + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 10 Data size: 1780 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Reducer 3 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/insert1_overwrite_partitions.q.out b/ql/src/test/results/clientpositive/llap/insert1_overwrite_partitions.q.out index db20610011fc..0474f0dd1adf 100644 --- a/ql/src/test/results/clientpositive/llap/insert1_overwrite_partitions.q.out +++ b/ql/src/test/results/clientpositive/llap/insert1_overwrite_partitions.q.out @@ -65,21 +65,16 @@ STAGE PLANS: alias: sourcetable filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -271,21 +266,16 @@ STAGE PLANS: alias: sourcetable filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -452,21 +442,16 @@ STAGE PLANS: alias: sourcetable filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/insert2_overwrite_partitions.q.out b/ql/src/test/results/clientpositive/llap/insert2_overwrite_partitions.q.out index 71e89cfc49a9..1ab56cf74c83 100644 --- a/ql/src/test/results/clientpositive/llap/insert2_overwrite_partitions.q.out +++ b/ql/src/test/results/clientpositive/llap/insert2_overwrite_partitions.q.out @@ -76,21 +76,16 @@ STAGE PLANS: alias: sourcetable filterExpr: (ds = '2011-11-11') (type: boolean) Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -235,21 +230,16 @@ STAGE PLANS: alias: sourcetable filterExpr: (ds = '2011-11-11') (type: boolean) Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/insert_into1.q.out b/ql/src/test/results/clientpositive/llap/insert_into1.q.out index 5febf3e4e93a..92d5c412ca75 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -40,22 +40,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -216,22 +211,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -392,22 +382,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/insert_into2.q.out b/ql/src/test/results/clientpositive/llap/insert_into2.q.out index 7e505e271cb1..b90595221b9f 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into2.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into2.q.out @@ -44,22 +44,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -269,22 +264,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -459,22 +449,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 50 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/insert_into3.q.out b/ql/src/test/results/clientpositive/llap/insert_into3.q.out index 9202e3b93b64..96760c9602f2 100644 --- a/ql/src/test/results/clientpositive/llap/insert_into3.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_into3.q.out @@ -62,36 +62,26 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 50 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + TopN Hash Memory Usage: 0.1 + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/jdbc_project_pushdown.q.out b/ql/src/test/results/clientpositive/llap/jdbc_project_pushdown.q.out index d3eb7a8cf7f4..2fe218a3ba95 100644 --- a/ql/src/test/results/clientpositive/llap/jdbc_project_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/jdbc_project_pushdown.q.out @@ -611,22 +611,17 @@ GROUP BY "t4"."fname" hive.sql.query.fieldTypes string,bigint hive.sql.query.split false Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: upper(fname) (type: string) - null sort order: z + Select Operator + expressions: $f1 (type: bigint), upper(fname) (type: string) + outputColumnNames: _col1, _col2 Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - top n: 5 - Select Operator - expressions: $f1 (type: bigint), upper(fname) (type: string) - outputColumnNames: _col1, _col2 + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col2 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: bigint) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index e1228b6403b9..6111b20e84d4 100644 --- a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -757,22 +757,17 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 1 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE @@ -1834,22 +1829,17 @@ STAGE PLANS: TableScan alias: src2 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 2 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Filter Operator predicate: key is not null (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out index 96372051bbd0..1745af157ef0 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out @@ -25,22 +25,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -122,22 +117,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: key (type: string) - null sort order: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -1208,6 +1198,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 value expressions: key (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1295,22 +1286,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 2.0E-5 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out index c5b053633129..ae458aa72ca5 100644 --- a/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_pushdown3.q.out @@ -25,22 +25,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -122,22 +117,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: key (type: string) - null sort order: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -1125,6 +1115,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 value expressions: key (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1222,22 +1213,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 2.0E-5 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out index caec28516823..3e4984936014 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_rewrite_8.q.out @@ -254,22 +254,17 @@ STAGE PLANS: TableScan alias: default.source_table_001_mv Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: my_id2 (type: bigint) - null sort order: z + Select Operator + expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: down_volume_sum (type: bigint), my_date (type: date), my_id2 (type: bigint), environment (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col2 (type: bigint) + null sort order: z + sort order: + Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col2 (type: bigint) - null sort order: z - sort order: + - Statistics: Num rows: 1 Data size: 159 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint), _col1 (type: date), _col3 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: bigint), _col1 (type: date), _col3 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out index 52ad5ad7cc13..a54edb69c3bc 100644 --- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out +++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out @@ -25,22 +25,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 30 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -123,22 +118,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: key (type: string) - null sort order: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 30 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.3 + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -1064,6 +1054,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 value expressions: key (type: string) Execution mode: llap LLAP IO: all inputs @@ -1152,22 +1143,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 2.0E-5 + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -1493,6 +1479,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 2.0E-5 Execution mode: llap LLAP IO: all inputs Map 4 @@ -1516,6 +1503,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 2.0E-5 Execution mode: llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/orc_createas1.q.out b/ql/src/test/results/clientpositive/llap/orc_createas1.q.out index 06f314671bb3..9094ef2d7b73 100644 --- a/ql/src/test/results/clientpositive/llap/orc_createas1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_createas1.q.out @@ -199,22 +199,17 @@ STAGE PLANS: TableScan alias: orc_createas1b Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 5 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out index dcc7c103b771..3dad42580a57 100644 --- a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out @@ -836,22 +836,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 0Y) and (d >= 10.0D) and (d < 12.0D) and UDFToInteger(si) BETWEEN 300 AND 400 and (s like '%son') and (not (s like '%car%'))) (type: boolean) Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -925,22 +920,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 0Y) and (d >= 10.0D) and (d < 12.0D) and UDFToInteger(si) BETWEEN 300 AND 400 and (s like '%son') and (not (s like '%car%'))) (type: boolean) Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -1080,22 +1070,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 10Y) and (t <> 101Y) and (d >= 10) and (d < 12.0D) and (s like '%son') and (not (s like '%car%')) and (t > 0Y) and si BETWEEN 300 AND 400) (type: boolean) Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -1104,22 +1089,17 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: KEY.reducesinkkey0 (type: string) - null sort order: a + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1196,22 +1176,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 10Y) and (t <> 101Y) and (d >= 10) and (d < 12.0D) and (s like '%son') and (not (s like '%car%')) and (t > 0Y) and si BETWEEN 300 AND 400) (type: boolean) Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -1220,22 +1195,17 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: KEY.reducesinkkey0 (type: string) - null sort order: a + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Reducer 3 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/order.q.out b/ql/src/test/results/clientpositive/llap/order.q.out index 03000ed7c77f..418bc94908cb 100644 --- a/ql/src/test/results/clientpositive/llap/order.q.out +++ b/ql/src/test/results/clientpositive/llap/order.q.out @@ -25,22 +25,17 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -112,22 +107,17 @@ STAGE PLANS: TableScan alias: x Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: key (type: string) - null sort order: a + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out index d7a825b592a6..72889aae4518 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out @@ -770,22 +770,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 0Y) and (d >= 10.0D) and (d < 12.0D) and UDFToInteger(si) BETWEEN 300 AND 400 and (s like '%son') and (not (s like '%car%'))) (type: boolean) Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -859,22 +854,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 0Y) and (d >= 10.0D) and (d < 12.0D) and UDFToInteger(si) BETWEEN 300 AND 400 and (s like '%son') and (not (s like '%car%'))) (type: boolean) Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 25 Data size: 2825 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -1059,22 +1049,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 10Y) and (t <> 101Y) and (d >= 10) and (d < 12.0D) and (s like '%son') and (not (s like '%car%')) and (t > 0Y) and si BETWEEN 300 AND 400) (type: boolean) Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -1083,22 +1068,17 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: KEY.reducesinkkey0 (type: string) - null sort order: a + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1175,22 +1155,17 @@ STAGE PLANS: Filter Operator predicate: ((t > 10Y) and (t <> 101Y) and (d >= 10) and (d < 12.0D) and (s like '%son') and (not (s like '%car%')) and (t > 0Y) and si BETWEEN 300 AND 400) (type: boolean) Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: s (type: string) - null sort order: a + Select Operator + expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: t (type: tinyint), si (type: smallint), d (type: double), s (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 44 Data size: 4972 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -1199,22 +1174,17 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: KEY.reducesinkkey0 (type: string) - null sort order: a + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), VALUE._col2 (type: double), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 + Reduce Output Operator + key expressions: _col3 (type: string) + null sort order: a + sort order: - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col3 (type: string) - null sort order: a - sort order: - - Statistics: Num rows: 3 Data size: 339 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: double) Reducer 3 Execution mode: llap Reduce Operator Tree: @@ -1303,22 +1273,17 @@ STAGE PLANS: Filter Operator predicate: ((f < 123.2) and (f > 1.92) and (f >= 9.99) and f BETWEEN 1.92 AND 123.2 and (i < 67627) and (i > 60627) and (i >= 60626) and i BETWEEN 60626 AND 67627 and (b < 4294967861L) and (b > 4294967261L) and (b >= 4294967260L) and b BETWEEN 4294967261L AND 4294967861L) (type: boolean) Statistics: Num rows: 911 Data size: 14576 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: f (type: float) - null sort order: a + Select Operator + expressions: f (type: float), i (type: int), b (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 911 Data size: 14576 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: f (type: float), i (type: int), b (type: bigint) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: float) + null sort order: a + sort order: - Statistics: Num rows: 911 Data size: 14576 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: float) - null sort order: a - sort order: - - Statistics: Num rows: 911 Data size: 14576 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: all inputs (cache only) Reducer 2 @@ -1327,22 +1292,17 @@ STAGE PLANS: Limit Number of rows: 3 Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: - - keys: KEY.reducesinkkey0 (type: float) - null sort order: a + Select Operator + expressions: KEY.reducesinkkey0 (type: float), VALUE._col0 (type: int), VALUE._col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: KEY.reducesinkkey0 (type: float), VALUE._col0 (type: int), VALUE._col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: _col0 (type: float) + null sort order: a + sort order: - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: float) - null sort order: a - sort order: - - Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: int), _col2 (type: bigint) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: int), _col2 (type: bigint) Reducer 3 Execution mode: llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out index 02d8275b94e8..00747b671473 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_7.q.out @@ -87,34 +87,25 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 14:double, val -28815.0)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 15:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 15:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0)) predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28815.0D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++ - keys: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 10:boolean, col 3:bigint, col 1:smallint, col 0:tinyint, col 8:timestamp, col 6:string, LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Select Operator + expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 16, 17, 18, 19, 21, 23, 24, 25, 27] + selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) + null sort order: zzzzzzzzzzzzzzz + sort order: +++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 20, 22, 26, 28, 30, 32, 33, 34, 36] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 20:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 22:int, LongColUnaryMinus(col 1:smallint) -> 26:smallint, LongColUnaryMinus(col 0:tinyint) -> 28:tinyint, LongColAddLongScalar(col 29:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 29:tinyint) -> 30:int, LongColMultiplyLongColumn(col 3:bigint, col 31:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 31:smallint) -> 32:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 33:int, LongColUnaryMinus(col 0:tinyint) -> 34:tinyint, LongColModuloLongColumn(col 35:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 35:tinyint) -> 36:tinyint - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - sort order: +++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -142,19 +133,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14] - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -336,34 +327,25 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 14:double, val -28792.315)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 15:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 15:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0)) predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28792.315D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++ - keys: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 10:boolean, col 3:bigint, col 1:smallint, col 0:tinyint, col 8:timestamp, col 6:string, LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Select Operator + expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 16, 17, 18, 19, 21, 23, 24, 25, 27] + selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) + null sort order: zzzzzzzzzzzzzzz + sort order: +++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 20, 22, 26, 28, 30, 32, 33, 34, 36] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 20:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 22:int, LongColUnaryMinus(col 1:smallint) -> 26:smallint, LongColUnaryMinus(col 0:tinyint) -> 28:tinyint, LongColAddLongScalar(col 29:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 29:tinyint) -> 30:int, LongColMultiplyLongColumn(col 3:bigint, col 31:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 31:smallint) -> 32:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 33:int, LongColUnaryMinus(col 0:tinyint) -> 34:tinyint, LongColModuloLongColumn(col 35:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 35:tinyint) -> 36:tinyint - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - sort order: +++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -391,19 +373,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14] - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out index 22a2e1922e37..2ed80593599b 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_8.q.out @@ -83,34 +83,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterDoubleColNotEqualDoubleScalar(col 15:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 15:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean))) predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean) Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++++++++++++++ - keys: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 8:timestamp, col 5:double, col 10:boolean, col 6:string, col 4:float, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Select Operator + expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 5, 10, 6, 4, 16, 17, 18, 20, 23, 24, 25, 26, 30] + selectExpressions: DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [8, 5, 10, 6, 4, 14, 15, 19, 22, 28, 21, 27, 29, 34] - selectExpressions: DoubleColUnaryMinus(col 5:double) -> 14:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 15:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 19:double, DoubleColAddDoubleColumn(col 21:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 21:float) -> 22:float, DoubleColAddDoubleColumn(col 21:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 5:double) -> 21:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 27:float, DoubleColUnaryMinus(col 4:float) -> 29:float, DoubleColAddDoubleColumn(col 31:double, col 33:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 31:double, DoubleColAddDoubleColumn(col 32:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 32:float) -> 33:float) -> 34:double - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -138,19 +129,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13] - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -319,34 +310,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterDoubleColNotEqualDoubleScalar(col 15:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 15:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean))) predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean) Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++++++++++++++ - keys: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 8:timestamp, col 5:double, col 10:boolean, col 6:string, col 4:float, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Select Operator + expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 5, 10, 6, 4, 16, 17, 18, 20, 23, 24, 25, 26, 30] + selectExpressions: DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [8, 5, 10, 6, 4, 14, 15, 19, 22, 28, 21, 27, 29, 34] - selectExpressions: DoubleColUnaryMinus(col 5:double) -> 14:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 15:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 19:double, DoubleColAddDoubleColumn(col 21:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 21:float) -> 22:float, DoubleColAddDoubleColumn(col 21:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 5:double) -> 21:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 27:float, DoubleColUnaryMinus(col 4:float) -> 29:float, DoubleColAddDoubleColumn(col 31:double, col 33:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 31:double, DoubleColAddDoubleColumn(col 32:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 32:float) -> 33:float) -> 34:double - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -374,19 +356,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13] - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out index 9f9a40bac222..bd3a9d35409a 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_div0.q.out @@ -222,35 +222,26 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterLongColGreaterLongScalar(col 3:bigint, val 0), FilterLongColLessLongScalar(col 3:bigint, val 100000000)) predicate: ((cbigint > 0L) and (cbigint < 100000000L)) (type: boolean) Statistics: Num rows: 3215 Data size: 38416 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double) - null sort order: zz - Statistics: Num rows: 3215 Data size: 38416 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 14:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint) -> 16:double) -> 17:double + Select Operator + expressions: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21)) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21)) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [14, 17, 20] + selectExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 14:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint) -> 16:double) -> 17:double, DecimalScalarDivideDecimalColumn(val 1.2, col 19:decimal(19,0))(children: CastLongToDecimal(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 19:decimal(19,0)) -> 20:decimal(22,21) + Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: double) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [15, 19, 22] - selectExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 16:double) -> 19:double, DecimalScalarDivideDecimalColumn(val 1.2, col 21:decimal(19,0))(children: CastLongToDecimal(col 20:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 20:bigint) -> 21:decimal(19,0)) -> 22:decimal(22,21) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: double) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: decimal(22,21)) + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: decimal(22,21)) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -455,35 +446,26 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -500.0), FilterDoubleColLessDoubleScalar(col 5:double, val -199.0)) predicate: ((cdouble >= -500.0D) and (cdouble < -199.0D)) (type: boolean) Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double) - null sort order: zz - Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 14:double, DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 17:double + Select Operator + expressions: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double) + outputColumnNames: _col0, _col1, _col2, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double) - outputColumnNames: _col0, _col1, _col2, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [14, 17, 20, 22, 24] + selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 14:double, DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 17:double, DoubleColDivideDoubleColumn(col 18:double, col 19:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 19:double) -> 20:double, DoubleScalarDivideDoubleColumn(val 3.0, col 21:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 21:double) -> 22:double, DoubleScalarDivideDoubleColumn(val 1.2, col 23:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 23:double) -> 24:double + Statistics: Num rows: 20 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [15, 19, 21, 22, 23] - selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 15:double, DoubleColDivideDoubleColumn(col 16:double, col 18:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double) -> 19:double, DoubleColDivideDoubleColumn(col 16:double, col 20:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 20:double) -> 21:double, DoubleScalarDivideDoubleColumn(val 3.0, col 16:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 22:double, DoubleScalarDivideDoubleColumn(val 1.2, col 16:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 23:double + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 20 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 20 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double) + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out index 1167c5d43e92..a5e0feef9a3c 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_limit.q.out @@ -32,21 +32,16 @@ STAGE PLANS: Filter Operator predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: cbigint (type: bigint), cdouble (type: double) - null sort order: zz - Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE - top n: 7 - Select Operator - expressions: cbigint (type: bigint), cdouble (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: double) - null sort order: zz - sort order: ++ - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: double) + null sort order: zz + sort order: ++ + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -70,13 +65,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 7 - Statistics: Num rows: 7 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -145,33 +140,24 @@ STAGE PLANS: predicateExpression: SelectColumnIsNotNull(col 0:tinyint) predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++ - keys: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - null sort order: zzz - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:tinyint, col 5:double, col 1:smallint + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 5, 1] + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double), _col2 (type: smallint) + null sort order: zzz + sort order: +++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 5, 1] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double), _col2 (type: smallint) - null sort order: zzz - sort order: +++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -199,19 +185,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out index a65dddfbdcd6..0d473d26de06 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_offset_limit.q.out @@ -139,34 +139,25 @@ STAGE PLANS: predicateExpression: SelectColumnIsNotNull(col 0:tinyint) predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: ctinyint (type: tinyint), cdouble (type: double) - null sort order: zz - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - top n: 13 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:tinyint, col 5:double + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 5, 1] + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 5, 1] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: smallint) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -194,20 +185,20 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Limit Vectorization: className: VectorLimitOperator native: true Offset of rows: 10 - Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out b/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out index 4f13a49cc63a..d974b769a783 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_vectorization_part_project.q.out @@ -79,21 +79,16 @@ STAGE PLANS: TableScan alias: alltypesparquet_part_n0 Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: (cdouble + 2.0D) (type: double) - null sort order: z + Select Operator + expressions: (cdouble + 2.0D) (type: double) + outputColumnNames: _col0 Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: (cdouble + 2.0D) (type: double) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/pcr.q.out b/ql/src/test/results/clientpositive/llap/pcr.q.out index a1f5a0f77d09..20abba15a828 100644 --- a/ql/src/test/results/clientpositive/llap/pcr.q.out +++ b/ql/src/test/results/clientpositive/llap/pcr.q.out @@ -4166,26 +4166,22 @@ STAGE PLANS: filterExpr: ((ds = '2008-04-08') and (11.0D = 11.0D)) (type: boolean) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE GatherStats: false - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + bucketingVersion: 2 + key expressions: _col0 (type: string) + null sort order: z + numBuckets: -1 + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - bucketingVersion: 2 - key expressions: _col0 (type: string) - null sort order: z - numBuckets: -1 - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: string) - auto parallelism: false + tag: -1 + TopN: 10 + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) + auto parallelism: false Execution mode: vectorized, llap LLAP IO: all inputs Path -> Alias: diff --git a/ql/src/test/results/clientpositive/llap/prepare_plan.q.out b/ql/src/test/results/clientpositive/llap/prepare_plan.q.out index 60ccab7e26a0..39109408fb11 100644 --- a/ql/src/test/results/clientpositive/llap/prepare_plan.q.out +++ b/ql/src/test/results/clientpositive/llap/prepare_plan.q.out @@ -241,22 +241,17 @@ STAGE PLANS: Filter Operator predicate: (key > CAST( $1 AS STRING)) (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -312,22 +307,17 @@ STAGE PLANS: Filter Operator predicate: (key > CAST( '100' AS STRING)) (type: boolean) Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 4 diff --git a/ql/src/test/results/clientpositive/llap/regex_col.q.out b/ql/src/test/results/clientpositive/llap/regex_col.q.out index cde71d60a651..d996e60cb322 100644 --- a/ql/src/test/results/clientpositive/llap/regex_col.q.out +++ b/ql/src/test/results/clientpositive/llap/regex_col.q.out @@ -506,21 +506,16 @@ STAGE PLANS: TableScan alias: srcpart Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: key (type: string), value (type: string) - null sort order: zz + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/select_as_omitted.q.out b/ql/src/test/results/clientpositive/llap/select_as_omitted.q.out index 02dd551fe6b6..494be94534d8 100644 --- a/ql/src/test/results/clientpositive/llap/select_as_omitted.q.out +++ b/ql/src/test/results/clientpositive/llap/select_as_omitted.q.out @@ -33,22 +33,17 @@ STAGE PLANS: TableScan alias: src Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - top n: 1 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/subquery_join_rewrite.q.out b/ql/src/test/results/clientpositive/llap/subquery_join_rewrite.q.out index 80221a3854bf..130d38c75c32 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_join_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_join_rewrite.q.out @@ -339,7 +339,7 @@ where ws1.ws_order_number in (select ws2.ws_order_number from web_sales ws2 orde POSTHOOK: type: QUERY POSTHOOK: Input: default@web_sales #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain cbo select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls last limit 1) @@ -373,7 +373,7 @@ HiveProject(ws1.ws_order_number=[$0], ws1.ws_warehouse_sk=[$1]) HiveProject(ws_order_number=[$0]) HiveTableScan(table=[[default, web_sales]], table:alias=[ws2]) -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls last limit 1) PREHOOK: type: QUERY @@ -386,7 +386,7 @@ POSTHOOK: Input: default@web_sales #### A masked pattern was here #### 2 1 2 2 -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain cbo select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls first limit 1) @@ -420,7 +420,7 @@ HiveProject(ws1.ws_order_number=[$0], ws1.ws_warehouse_sk=[$1]) HiveProject(ws_order_number=[$0]) HiveTableScan(table=[[default, web_sales]], table:alias=[ws2]) -Warning: Shuffle Join MERGEJOIN[36][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls first limit 1) PREHOOK: type: QUERY @@ -431,7 +431,7 @@ where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 POSTHOOK: type: QUERY POSTHOOK: Input: default@web_sales #### A masked pattern was here #### -Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain cbo select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls last limit 1 offset 2) @@ -464,7 +464,7 @@ HiveProject(ws1.ws_order_number=[$0], ws1.ws_warehouse_sk=[$1]) HiveProject(ws_order_number=[$0]) HiveTableScan(table=[[default, web_sales]], table:alias=[ws2]) -Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[32][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select * from web_sales ws1 where ws1.ws_order_number not in (select ws2.ws_order_number from web_sales ws2 order by ws2.ws_order_number nulls last limit 1 offset 2) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index fa79cd87dd13..88e0ee66f049 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -832,7 +832,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select p_name, p_size from part where part.p_size not in diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index 69beb9d179b0..921fce08b28e 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -2908,7 +2908,7 @@ POSTHOOK: Input: default@part 6 false 6 false 7 false -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select p_type, (select p_size from part order by p_size limit 1) = 1 from part order by p_type PREHOOK: type: QUERY PREHOOK: Input: default@part @@ -2945,21 +2945,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: string) - Top N Key Operator - sort order: + - keys: p_size (type: int) - null sort order: z + Select Operator + expressions: p_size (type: int) + outputColumnNames: _col0 Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - top n: 1 - Select Operator - expressions: p_size (type: int) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -3019,7 +3014,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join MERGEJOIN[16][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select p_type, (select p_size from part order by p_size limit 1) = 1 from part order by p_type PREHOOK: type: QUERY PREHOOK: Input: default@part diff --git a/ql/src/test/results/clientpositive/llap/temp_table.q.out b/ql/src/test/results/clientpositive/llap/temp_table.q.out index 00e5c1cb61ee..84bb4dafa6a3 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -227,22 +227,17 @@ STAGE PLANS: TableScan alias: foo Statistics: Num rows: 247 Data size: 86848 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: key (type: string) - null sort order: z + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 247 Data size: 86848 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + Statistics: Num rows: 247 Data size: 86848 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 247 Data size: 86848 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -320,18 +315,13 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 247 Data size: 86848 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 175904 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 175904 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Map 4 @@ -343,18 +333,13 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 253 Data size: 89056 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: _col0 (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 175904 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 175904 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 3 diff --git a/ql/src/test/results/clientpositive/llap/temp_table_insert1_overwrite_partitions.q.out b/ql/src/test/results/clientpositive/llap/temp_table_insert1_overwrite_partitions.q.out index cda8f52fda10..4ebe57a2cf3b 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table_insert1_overwrite_partitions.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table_insert1_overwrite_partitions.q.out @@ -65,21 +65,16 @@ STAGE PLANS: alias: sourcetable_temp filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -270,21 +265,16 @@ STAGE PLANS: alias: sourcetable_temp filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -413,21 +403,16 @@ STAGE PLANS: alias: sourcetable_temp filterExpr: ((ds = '2011-11-11') and (hr = '11')) (type: boolean) Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 98 Data size: 31280 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/temp_table_insert2_overwrite_partitions.q.out b/ql/src/test/results/clientpositive/llap/temp_table_insert2_overwrite_partitions.q.out index 841bd4296c20..7c3753ccdc33 100644 --- a/ql/src/test/results/clientpositive/llap/temp_table_insert2_overwrite_partitions.q.out +++ b/ql/src/test/results/clientpositive/llap/temp_table_insert2_overwrite_partitions.q.out @@ -76,21 +76,16 @@ STAGE PLANS: alias: sourcetable_temp filterExpr: (ds = '2011-11-11') (type: boolean) Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 @@ -235,21 +230,16 @@ STAGE PLANS: alias: sourcetable_temp filterExpr: (ds = '2011-11-11') (type: boolean) Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Top N Key Operator - sort order: -- - keys: one (type: string), two (type: string) - null sort order: aa + Select Operator + expressions: one (type: string), two (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - top n: 5 - Select Operator - expressions: one (type: string), two (type: string) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: -- Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: -- - Statistics: Num rows: 123 Data size: 40112 Basic stats: PARTIAL Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/topnkey.q.out b/ql/src/test/results/clientpositive/llap/topnkey.q.out index e72e3cc6d841..148e4e0df611 100644 --- a/ql/src/test/results/clientpositive/llap/topnkey.q.out +++ b/ql/src/test/results/clientpositive/llap/topnkey.q.out @@ -497,21 +497,16 @@ STAGE PLANS: TableScan alias: t_test Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: a (type: int), b (type: int) - null sort order: zz + Select Operator + expressions: a (type: int), b (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: a (type: int), b (type: int) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: int) + null sort order: zz + sort order: ++ Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) - null sort order: zz - sort order: ++ - Statistics: Num rows: 8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/udtf_explode.q.out b/ql/src/test/results/clientpositive/llap/udtf_explode.q.out index 94d3cd930a48..ab77f2f8e3f4 100644 --- a/ql/src/test/results/clientpositive/llap/udtf_explode.q.out +++ b/ql/src/test/results/clientpositive/llap/udtf_explode.q.out @@ -275,21 +275,16 @@ STAGE PLANS: UDTF Operator Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE function name: explode - Top N Key Operator - sort order: ++ - keys: key (type: int), value (type: string) - null sort order: zz - Statistics: Num rows: 500 Data size: 259500 Basic stats: COMPLETE Column stats: COMPLETE - top n: 3 - Select Operator - expressions: key (type: int), value (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 500 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: llap LLAP IO: all inputs Reducer 2 diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out index 142ce776180f..03b02d30ad9c 100644 --- a/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_case_when_2.q.out @@ -975,37 +975,28 @@ STAGE PLANS: TableScan Vectorization: native: true vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] - Top N Key Operator - sort order: + - keys: q548284 (type: int) - null sort order: z - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - top n: 1 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int + Select Operator + expressions: q548284 (type: int), CASE WHEN ((q548284 = 1)) THEN (0.2) WHEN ((q548284 = 2)) THEN (0.4) WHEN ((q548284 = 3)) THEN (0.6) WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (null) END (type: decimal(2,1)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: q548284 (type: int), CASE WHEN ((q548284 = 1)) THEN (0.2) WHEN ((q548284 = 2)) THEN (0.4) WHEN ((q548284 = 3)) THEN (0.6) WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (null) END (type: decimal(2,1)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 17] + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(2,1), col 16:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 1) -> 3:boolean, ConstantVectorExpression(val 0.2) -> 4:decimal(2,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(2,1), col 15:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 2) -> 5:boolean, ConstantVectorExpression(val 0.4) -> 6:decimal(2,1), IfExprCondExprCondExpr(col 7:boolean, col 8:decimal(2,1), col 14:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 3) -> 7:boolean, ConstantVectorExpression(val 0.6) -> 8:decimal(2,1), IfExprCondExprCondExpr(col 9:boolean, col 10:decimal(2,1), col 13:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 9:boolean, ConstantVectorExpression(val 0.8) -> 10:decimal(2,1), IfExprCondExprNull(col 11:boolean, col 12:decimal(2,1), null)(children: LongColEqualLongScalar(col 0:int, val 5) -> 11:boolean, ConstantVectorExpression(val 1) -> 12:decimal(2,1)) -> 13:decimal(2,1)) -> 14:decimal(2,1)) -> 15:decimal(2,1)) -> 16:decimal(2,1)) -> 17:decimal(2,1) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:int native: true - projectedOutputColumnNums: [0, 17] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(2,1), col 16:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 1) -> 3:boolean, ConstantVectorExpression(val 0.2) -> 4:decimal(2,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(2,1), col 15:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 2) -> 5:boolean, ConstantVectorExpression(val 0.4) -> 6:decimal(2,1), IfExprCondExprCondExpr(col 7:boolean, col 8:decimal(2,1), col 14:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 3) -> 7:boolean, ConstantVectorExpression(val 0.6) -> 8:decimal(2,1), IfExprCondExprCondExpr(col 9:boolean, col 10:decimal(2,1), col 13:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 9:boolean, ConstantVectorExpression(val 0.8) -> 10:decimal(2,1), IfExprCondExprNull(col 11:boolean, col 12:decimal(2,1), null)(children: LongColEqualLongScalar(col 0:int, val 5) -> 11:boolean, ConstantVectorExpression(val 1) -> 12:decimal(2,1)) -> 13:decimal(2,1)) -> 14:decimal(2,1)) -> 15:decimal(2,1)) -> 16:decimal(2,1)) -> 17:decimal(2,1) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 17:decimal(2,1) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 17:decimal(2,1) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(2,1)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(2,1)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1119,37 +1110,28 @@ STAGE PLANS: TableScan Vectorization: native: true vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] - Top N Key Operator - sort order: + - keys: q548284 (type: int) - null sort order: z - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - top n: 1 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int + Select Operator + expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(2,1)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(2,1)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 9] + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(2,1), col 8:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 3:boolean, ConstantVectorExpression(val 0.8) -> 4:decimal(2,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(2,1), col 7:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 5) -> 5:boolean, ConstantVectorExpression(val 1) -> 6:decimal(2,1), ConstantVectorExpression(val 8) -> 7:decimal(2,1)) -> 8:decimal(2,1)) -> 9:decimal(2,1) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:int native: true - projectedOutputColumnNums: [0, 9] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(2,1), col 8:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 3:boolean, ConstantVectorExpression(val 0.8) -> 4:decimal(2,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(2,1), col 7:decimal(2,1))(children: LongColEqualLongScalar(col 0:int, val 5) -> 5:boolean, ConstantVectorExpression(val 1) -> 6:decimal(2,1), ConstantVectorExpression(val 8) -> 7:decimal(2,1)) -> 8:decimal(2,1)) -> 9:decimal(2,1) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 9:decimal(2,1) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 9:decimal(2,1) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(2,1)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(2,1)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1263,37 +1245,28 @@ STAGE PLANS: TableScan Vectorization: native: true vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] - Top N Key Operator - sort order: + - keys: q548284 (type: int) - null sort order: z - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - top n: 1 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int + Select Operator + expressions: q548284 (type: int), CASE WHEN ((q548284 = 1)) THEN (0.2) WHEN ((q548284 = 2)) THEN (0.4) WHEN ((q548284 = 3)) THEN (0.6) WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (null) END (type: decimal(11,1)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: q548284 (type: int), CASE WHEN ((q548284 = 1)) THEN (0.2) WHEN ((q548284 = 2)) THEN (0.4) WHEN ((q548284 = 3)) THEN (0.6) WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (null) END (type: decimal(11,1)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 17] + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(11,1), col 16:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 1) -> 3:boolean, ConstantVectorExpression(val 0.2) -> 4:decimal(11,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(11,1), col 15:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 2) -> 5:boolean, ConstantVectorExpression(val 0.4) -> 6:decimal(11,1), IfExprCondExprCondExpr(col 7:boolean, col 8:decimal(11,1), col 14:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 3) -> 7:boolean, ConstantVectorExpression(val 0.6) -> 8:decimal(11,1), IfExprCondExprCondExpr(col 9:boolean, col 10:decimal(11,1), col 13:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 9:boolean, ConstantVectorExpression(val 0.8) -> 10:decimal(11,1), IfExprCondExprNull(col 11:boolean, col 12:decimal(11,1), null)(children: LongColEqualLongScalar(col 0:int, val 5) -> 11:boolean, ConstantVectorExpression(val 1) -> 12:decimal(11,1)) -> 13:decimal(11,1)) -> 14:decimal(11,1)) -> 15:decimal(11,1)) -> 16:decimal(11,1)) -> 17:decimal(11,1) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:int native: true - projectedOutputColumnNums: [0, 17] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(11,1), col 16:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 1) -> 3:boolean, ConstantVectorExpression(val 0.2) -> 4:decimal(11,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(11,1), col 15:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 2) -> 5:boolean, ConstantVectorExpression(val 0.4) -> 6:decimal(11,1), IfExprCondExprCondExpr(col 7:boolean, col 8:decimal(11,1), col 14:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 3) -> 7:boolean, ConstantVectorExpression(val 0.6) -> 8:decimal(11,1), IfExprCondExprCondExpr(col 9:boolean, col 10:decimal(11,1), col 13:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 9:boolean, ConstantVectorExpression(val 0.8) -> 10:decimal(11,1), IfExprCondExprNull(col 11:boolean, col 12:decimal(11,1), null)(children: LongColEqualLongScalar(col 0:int, val 5) -> 11:boolean, ConstantVectorExpression(val 1) -> 12:decimal(11,1)) -> 13:decimal(11,1)) -> 14:decimal(11,1)) -> 15:decimal(11,1)) -> 16:decimal(11,1)) -> 17:decimal(11,1) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 17:decimal(11,1) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 17:decimal(11,1) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(11,1)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(11,1)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1407,37 +1380,28 @@ STAGE PLANS: TableScan Vectorization: native: true vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] - Top N Key Operator - sort order: + - keys: q548284 (type: int) - null sort order: z - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - top n: 1 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int + Select Operator + expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(11,1)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(11,1)) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 9] + selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(11,1), col 8:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 3:boolean, ConstantVectorExpression(val 0.8) -> 4:decimal(11,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(11,1), col 7:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 5) -> 5:boolean, ConstantVectorExpression(val 1) -> 6:decimal(11,1), ConstantVectorExpression(val 8) -> 7:decimal(11,1)) -> 8:decimal(11,1)) -> 9:decimal(11,1) + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:int native: true - projectedOutputColumnNums: [0, 9] - selectExpressions: IfExprCondExprCondExpr(col 3:boolean, col 4:decimal(11,1), col 8:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 4) -> 3:boolean, ConstantVectorExpression(val 0.8) -> 4:decimal(11,1), IfExprCondExprCondExpr(col 5:boolean, col 6:decimal(11,1), col 7:decimal(11,1))(children: LongColEqualLongScalar(col 0:int, val 5) -> 5:boolean, ConstantVectorExpression(val 1) -> 6:decimal(11,1), ConstantVectorExpression(val 8) -> 7:decimal(11,1)) -> 8:decimal(11,1)) -> 9:decimal(11,1) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 9:decimal(11,1) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 9:decimal(11,1) - Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: decimal(11,1)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: decimal(11,1)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out b/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out index 5f8bab6d6380..4dc7e85693cb 100644 --- a/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_case_when_conversion.q.out @@ -52,19 +52,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsNull(col 5:double) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 6:string, col 2:int, col 4:float, col 1:smallint, IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 24:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 23:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 22:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, IfExprCondExprColumn(col 19:boolean, col 20:string, col 21:string)(children: IsNotNull(col 1:smallint) -> 19:boolean, CastLongToString(col 1:smallint) -> 20:string, ConstantVectorExpression(val none) -> 21:string) -> 22:string) -> 23:string) -> 24:string) -> 25:string + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [6, 2, 4, 1, 25] + selectExpressions: IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 24:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 23:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 22:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, IfExprCondExprColumn(col 19:boolean, col 20:string, col 21:string)(children: IsNotNull(col 1:smallint) -> 19:boolean, CastLongToString(col 1:smallint) -> 20:string, ConstantVectorExpression(val none) -> 21:string) -> 22:string) -> 23:string) -> 24:string) -> 25:string + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [6, 2, 4, 1, 37] - selectExpressions: IfExprColumnCondExpr(col 26:boolean, col 6:stringcol 36:string)(children: IsNotNull(col 6:string) -> 26:boolean, col 6:string, IfExprCondExprCondExpr(col 27:boolean, col 28:string, col 35:string)(children: IsNotNull(col 2:int) -> 27:boolean, CastLongToString(col 2:int) -> 28:string, IfExprCondExprCondExpr(col 29:boolean, col 30:string, col 34:string)(children: IsNotNull(col 4:float) -> 29:boolean, CastFloatToString(col 4:float) -> 30:string, IfExprCondExprColumn(col 31:boolean, col 32:string, col 33:string)(children: IsNotNull(col 1:smallint) -> 31:boolean, CastLongToString(col 1:smallint) -> 32:string, ConstantVectorExpression(val none) -> 33:string) -> 34:string) -> 35:string) -> 36:string) -> 37:string - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -210,19 +206,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsTrue(col 14:boolean)(children: VectorUDFAdaptor(cdouble is null) -> 14:boolean) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 6:string, col 2:int, col 4:float, col 1:smallint, VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE ('none') END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 22:string + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [6, 2, 4, 1, 22] + selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE ('none') END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 22:string + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [6, 2, 4, 1, 23] - selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE ('none') END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 23:string - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -368,19 +360,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsNull(col 5:double) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 6:string, col 2:int, col 4:float, col 1:smallint, IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 23:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 22:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 21:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, IfExprCondExprNull(col 19:boolean, col 20:string, null)(children: IsNotNull(col 1:smallint) -> 19:boolean, CastLongToString(col 1:smallint) -> 20:string) -> 21:string) -> 22:string) -> 23:string) -> 24:string + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [6, 2, 4, 1, 24] + selectExpressions: IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 23:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 22:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 21:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, IfExprCondExprNull(col 19:boolean, col 20:string, null)(children: IsNotNull(col 1:smallint) -> 19:boolean, CastLongToString(col 1:smallint) -> 20:string) -> 21:string) -> 22:string) -> 23:string) -> 24:string + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [6, 2, 4, 1, 35] - selectExpressions: IfExprColumnCondExpr(col 25:boolean, col 6:stringcol 34:string)(children: IsNotNull(col 6:string) -> 25:boolean, col 6:string, IfExprCondExprCondExpr(col 26:boolean, col 27:string, col 33:string)(children: IsNotNull(col 2:int) -> 26:boolean, CastLongToString(col 2:int) -> 27:string, IfExprCondExprCondExpr(col 28:boolean, col 29:string, col 32:string)(children: IsNotNull(col 4:float) -> 28:boolean, CastFloatToString(col 4:float) -> 29:string, IfExprCondExprNull(col 30:boolean, col 31:string, null)(children: IsNotNull(col 1:smallint) -> 30:boolean, CastLongToString(col 1:smallint) -> 31:string) -> 32:string) -> 33:string) -> 34:string) -> 35:string - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -526,19 +514,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsTrue(col 14:boolean)(children: VectorUDFAdaptor(cdouble is null) -> 14:boolean) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 6:string, col 2:int, col 4:float, col 1:smallint, VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE (null) END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 22:string + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [6, 2, 4, 1, 22] + selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE (null) END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 22:string + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [6, 2, 4, 1, 23] - selectExpressions: VectorUDFAdaptor(CASE WHEN (cstring1 is not null) THEN (cstring1) WHEN (cint is not null) THEN (CAST( cint AS STRING)) WHEN (cfloat is not null) THEN (CAST( cfloat AS STRING)) WHEN (csmallint is not null) THEN (CAST( csmallint AS STRING)) ELSE (null) END)(children: VectorUDFAdaptor(cstring1 is not null) -> 15:boolean, VectorUDFAdaptor(cint is not null) -> 16:boolean, VectorUDFAdaptor(CAST( cint AS STRING)) -> 17:string, VectorUDFAdaptor(cfloat is not null) -> 18:boolean, VectorUDFAdaptor(CAST( cfloat AS STRING)) -> 19:string, VectorUDFAdaptor(csmallint is not null) -> 20:boolean, VectorUDFAdaptor(CAST( csmallint AS STRING)) -> 21:string) -> 23:string - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out index 188b71359225..22c24c9aad9f 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce.q.out @@ -36,19 +36,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsNull(col 5:double) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 6:string, col 2:int, col 4:float, col 1:smallint, IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 21:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 20:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 19:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, CastLongToString(col 1:smallint) -> 19:string) -> 20:string) -> 21:string) -> 22:string + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [6, 2, 4, 1, 22] + selectExpressions: IfExprColumnCondExpr(col 14:boolean, col 6:stringcol 21:string)(children: IsNotNull(col 6:string) -> 14:boolean, col 6:string, IfExprCondExprCondExpr(col 15:boolean, col 16:string, col 20:string)(children: IsNotNull(col 2:int) -> 15:boolean, CastLongToString(col 2:int) -> 16:string, IfExprCondExprCondExpr(col 17:boolean, col 18:string, col 19:string)(children: IsNotNull(col 4:float) -> 17:boolean, CastFloatToString(col 4:float) -> 18:string, CastLongToString(col 1:smallint) -> 19:string) -> 20:string) -> 21:string) -> 22:string + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [6, 2, 4, 1, 31] - selectExpressions: IfExprColumnCondExpr(col 23:boolean, col 6:stringcol 30:string)(children: IsNotNull(col 6:string) -> 23:boolean, col 6:string, IfExprCondExprCondExpr(col 24:boolean, col 25:string, col 29:string)(children: IsNotNull(col 2:int) -> 24:boolean, CastLongToString(col 2:int) -> 25:string, IfExprCondExprCondExpr(col 26:boolean, col 27:string, col 28:string)(children: IsNotNull(col 4:float) -> 26:boolean, CastFloatToString(col 4:float) -> 27:string, CastLongToString(col 1:smallint) -> 28:string) -> 29:string) -> 30:string) -> 31:string - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -152,19 +148,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: SelectColumnIsNull(col 0:tinyint) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 5:double, col 2:int, IfExprCondExprColumn(col 17:boolean, col 19:double, col 20:double)(children: ColAndCol(col 14:boolean, col 16:boolean)(children: IsNotNull(col 5:double) -> 14:boolean, IsNotNull(col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 16:boolean) -> 17:boolean, DoubleColAddDoubleColumn(col 5:double, col 18:double)(children: FuncLog2LongToDouble(col 2:int) -> 18:double) -> 19:double, ConstantVectorExpression(val 0.0) -> 20:double) -> 21:double + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [5, 2, 21] + selectExpressions: IfExprCondExprColumn(col 17:boolean, col 19:double, col 20:double)(children: ColAndCol(col 14:boolean, col 16:boolean)(children: IsNotNull(col 5:double) -> 14:boolean, IsNotNull(col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 16:boolean) -> 17:boolean, DoubleColAddDoubleColumn(col 5:double, col 18:double)(children: FuncLog2LongToDouble(col 2:int) -> 18:double) -> 19:double, ConstantVectorExpression(val 0.0) -> 20:double) -> 21:double + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [5, 2, 23] - selectExpressions: IfExprCondExprColumn(col 22:boolean, col 18:double, col 15:double)(children: ColAndCol(col 14:boolean, col 16:boolean)(children: IsNotNull(col 5:double) -> 14:boolean, IsNotNull(col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 16:boolean) -> 22:boolean, DoubleColAddDoubleColumn(col 5:double, col 15:double)(children: FuncLog2LongToDouble(col 2:int) -> 15:double) -> 18:double, ConstantVectorExpression(val 0.0) -> 15:double) -> 23:double - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -378,19 +370,15 @@ STAGE PLANS: className: VectorFilterOperator native: true predicateExpression: FilterExprOrExpr(children: SelectColumnIsNotNull(col 9:timestamp), SelectColumnIsNotNull(col 8:timestamp)) - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 8:timestamp, col 9:timestamp, IfExprTimestampColumnColumn(col 14:boolean, col 8:timestampcol 9:timestamp)(children: IsNotNull(col 8:timestamp) -> 14:boolean) -> 15:timestamp + Select Vectorization: + className: VectorSelectOperator native: true - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 9, 15] + selectExpressions: IfExprTimestampColumnColumn(col 14:boolean, col 8:timestampcol 9:timestamp)(children: IsNotNull(col 8:timestamp) -> 14:boolean) -> 15:timestamp + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [8, 9, 16] - selectExpressions: IfExprTimestampColumnColumn(col 14:boolean, col 8:timestampcol 9:timestamp)(children: IsNotNull(col 8:timestamp) -> 14:boolean) -> 16:timestamp - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_data_types.q.out b/ql/src/test/results/clientpositive/llap/vector_data_types.q.out index fa4218a5c970..9c7db8e0de1b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_data_types.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_data_types.q.out @@ -143,22 +143,17 @@ STAGE PLANS: TableScan alias: over1korc_n1 Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++ - keys: t (type: tinyint), si (type: smallint), i (type: int) - null sort order: zzz + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float), d (type: double), bo (type: boolean), s (type: string), ts (type: timestamp), dec (type: decimal(4,2)), bin (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Select Operator - expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float), d (type: double), bo (type: boolean), s (type: string), ts (type: timestamp), dec (type: decimal(4,2)), bin (type: binary) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int) - null sort order: zzz - sort order: +++ - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int) + null sort order: zzz + sort order: +++ + Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: llap LLAP IO: all inputs Reducer 2 @@ -167,7 +162,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: smallint), KEY.reducesinkkey2 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: float), VALUE._col2 (type: double), VALUE._col3 (type: boolean), VALUE._col4 (type: string), VALUE._col5 (type: timestamp), VALUE._col6 (type: decimal(4,2)), VALUE._col7 (type: binary) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Statistics: Num rows: 20 Data size: 6820 Basic stats: COMPLETE Column stats: COMPLETE @@ -255,34 +250,25 @@ STAGE PLANS: Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: +++ - keys: t (type: tinyint), si (type: smallint), i (type: int) - null sort order: zzz - Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:tinyint, col 1:smallint, col 2:int + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float), d (type: double), bo (type: boolean), s (type: string), ts (type: timestamp), dec (type: decimal(4,2)), bin (type: binary) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float), d (type: double), bo (type: boolean), s (type: string), ts (type: timestamp), dec (type: decimal(4,2)), bin (type: binary) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int) + null sort order: zzz + sort order: +++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int) - null sort order: zzz - sort order: +++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: boolean), _col7 (type: string), _col8 (type: timestamp), _col9 (type: decimal(4,2)), _col10 (type: binary) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -310,7 +296,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - Statistics: Num rows: 1050 Data size: 357661 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1050 Data size: 358026 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out index 5ad7725fc0c0..b6cf1009e40b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_expressions.q.out @@ -74,35 +74,26 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterDecimalColGreaterDecimalScalar(col 1:decimal(20,10), val 0), FilterDecimalColLessDecimalScalar(col 1:decimal(20,10), val 12345.5678), FilterDecimalColGreaterDecimalScalar(col 2:decimal(23,14), val 1000), SelectColumnIsNotNull(col 0:double)) predicate: ((cdecimal1 > 0) and (cdecimal1 < 12345.5678) and (cdecimal2 > 1000) and cdouble is not null) (type: boolean) Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: ++++++++++++++ - keys: (cdecimal1 + cdecimal2) (type: decimal(25,14)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(26,14)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(38,13)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(38,17)), (cdecimal1 % 10) (type: decimal(12,10)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: DecimalColAddDecimalColumn(col 1:decimal(20,10), col 2:decimal(23,14)) -> 5:decimal(25,14), DecimalColSubtractDecimalColumn(col 1:decimal(20,10), col 6:decimal(25,14))(children: DecimalScalarMultiplyDecimalColumn(val 2, col 2:decimal(23,14)) -> 6:decimal(25,14)) -> 7:decimal(26,14), DecimalColDivideDecimalColumn(col 8:decimal(21,10), col 2:decimal(23,14))(children: DecimalColAddDecimalScalar(col 1:decimal(20,10), val 2.34) -> 8:decimal(21,10)) -> 9:decimal(38,13), DecimalColMultiplyDecimalColumn(col 1:decimal(20,10), col 10:decimal(27,17))(children: DecimalColDivideDecimalScalar(col 2:decimal(23,14), val 3.4) -> 10:decimal(27,17)) -> 11:decimal(38,17), DecimalColModuloDecimalScalar(col 1:decimal(20,10), val 10) -> 12:decimal(12,10), CastDecimalToLong(col 1:decimal(20,10)) -> 13:int, CastDecimalToLong(col 2:decimal(23,14)) -> 14:smallint, CastDecimalToLong(col 2:decimal(23,14)) -> 15:tinyint, CastDecimalToLong(col 1:decimal(20,10)) -> 16:bigint, CastDecimalToBoolean(col 1:decimal(20,10)) -> 17:boolean, CastDecimalToDouble(col 2:decimal(23,14)) -> 18:double, CastDecimalToFloat(col 1:decimal(20,10)) -> 19:float, CastDecimalToString(col 2:decimal(23,14)) -> 20:string, CastDecimalToTimestamp(col 1:decimal(20,10)) -> 21:timestamp + Select Operator + expressions: (cdecimal1 + cdecimal2) (type: decimal(25,14)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(26,14)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(38,13)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(38,17)), (cdecimal1 % 10) (type: decimal(12,10)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cdecimal1 + cdecimal2) (type: decimal(25,14)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(26,14)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(38,13)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(38,17)), (cdecimal1 % 10) (type: decimal(12,10)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [5, 7, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21] + selectExpressions: DecimalColAddDecimalColumn(col 1:decimal(20,10), col 2:decimal(23,14)) -> 5:decimal(25,14), DecimalColSubtractDecimalColumn(col 1:decimal(20,10), col 6:decimal(25,14))(children: DecimalScalarMultiplyDecimalColumn(val 2, col 2:decimal(23,14)) -> 6:decimal(25,14)) -> 7:decimal(26,14), DecimalColDivideDecimalColumn(col 8:decimal(21,10), col 2:decimal(23,14))(children: DecimalColAddDecimalScalar(col 1:decimal(20,10), val 2.34) -> 8:decimal(21,10)) -> 9:decimal(38,13), DecimalColMultiplyDecimalColumn(col 1:decimal(20,10), col 10:decimal(27,17))(children: DecimalColDivideDecimalScalar(col 2:decimal(23,14), val 3.4) -> 10:decimal(27,17)) -> 11:decimal(38,17), DecimalColModuloDecimalScalar(col 1:decimal(20,10), val 10) -> 12:decimal(12,10), CastDecimalToLong(col 1:decimal(20,10)) -> 13:int, CastDecimalToLong(col 2:decimal(23,14)) -> 14:smallint, CastDecimalToLong(col 2:decimal(23,14)) -> 15:tinyint, CastDecimalToLong(col 1:decimal(20,10)) -> 16:bigint, CastDecimalToBoolean(col 1:decimal(20,10)) -> 17:boolean, CastDecimalToDouble(col 2:decimal(23,14)) -> 18:double, CastDecimalToFloat(col 1:decimal(20,10)) -> 19:float, CastDecimalToString(col 2:decimal(23,14)) -> 20:string, CastDecimalToTimestamp(col 1:decimal(20,10)) -> 21:timestamp + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(25,14)), _col1 (type: decimal(26,14)), _col2 (type: decimal(38,13)), _col3 (type: decimal(38,17)), _col4 (type: decimal(12,10)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 5:decimal(25,14), 7:decimal(26,14), 9:decimal(38,13), 11:decimal(38,17), 12:decimal(12,10), 13:int, 14:smallint, 15:tinyint, 16:bigint, 17:boolean, 18:double, 19:float, 20:string, 21:timestamp native: true - projectedOutputColumnNums: [6, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] - selectExpressions: DecimalColAddDecimalColumn(col 1:decimal(20,10), col 2:decimal(23,14)) -> 6:decimal(25,14), DecimalColSubtractDecimalColumn(col 1:decimal(20,10), col 22:decimal(25,14))(children: DecimalScalarMultiplyDecimalColumn(val 2, col 2:decimal(23,14)) -> 22:decimal(25,14)) -> 23:decimal(26,14), DecimalColDivideDecimalColumn(col 8:decimal(21,10), col 2:decimal(23,14))(children: DecimalColAddDecimalScalar(col 1:decimal(20,10), val 2.34) -> 8:decimal(21,10)) -> 24:decimal(38,13), DecimalColMultiplyDecimalColumn(col 1:decimal(20,10), col 10:decimal(27,17))(children: DecimalColDivideDecimalScalar(col 2:decimal(23,14), val 3.4) -> 10:decimal(27,17)) -> 25:decimal(38,17), DecimalColModuloDecimalScalar(col 1:decimal(20,10), val 10) -> 26:decimal(12,10), CastDecimalToLong(col 1:decimal(20,10)) -> 27:int, CastDecimalToLong(col 2:decimal(23,14)) -> 28:smallint, CastDecimalToLong(col 2:decimal(23,14)) -> 29:tinyint, CastDecimalToLong(col 1:decimal(20,10)) -> 30:bigint, CastDecimalToBoolean(col 1:decimal(20,10)) -> 31:boolean, CastDecimalToDouble(col 2:decimal(23,14)) -> 32:double, CastDecimalToFloat(col 1:decimal(20,10)) -> 33:float, CastDecimalToString(col 2:decimal(23,14)) -> 34:string, CastDecimalToTimestamp(col 1:decimal(20,10)) -> 35:timestamp + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: decimal(25,14)), _col1 (type: decimal(26,14)), _col2 (type: decimal(38,13)), _col3 (type: decimal(38,17)), _col4 (type: decimal(12,10)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 6:decimal(25,14), 23:decimal(26,14), 24:decimal(38,13), 25:decimal(38,17), 26:decimal(12,10), 27:int, 28:smallint, 29:tinyint, 30:bigint, 31:boolean, 32:double, 33:float, 34:string, 35:timestamp - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -119,7 +110,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(20,10), cdecimal2:decimal(23,14) partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(25,14), decimal(25,14), decimal(26,14), decimal(21,10), decimal(38,13), decimal(27,17), decimal(38,17), decimal(12,10), bigint, bigint, bigint, bigint, bigint, double, double, string, timestamp, decimal(25,14), decimal(26,14), decimal(38,13), decimal(38,17), decimal(12,10), bigint, bigint, bigint, bigint, bigint, double, double, string, timestamp] + scratchColumnTypeNames: [decimal(25,14), decimal(25,14), decimal(26,14), decimal(21,10), decimal(38,13), decimal(27,17), decimal(38,17), decimal(12,10), bigint, bigint, bigint, bigint, bigint, double, double, string, timestamp] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -261,35 +252,26 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterDecimal64ColGreaterDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, val 0), FilterDecimalColLessDecimalScalar(col 5:decimal(10,3), val 12345.5678)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)), FilterDecimal64ColGreaterDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, val 100000), SelectColumnIsNotNull(col 0:double)) predicate: ((cdecimal1 > 0) and (cdecimal1 < 12345.5678) and (cdecimal2 > 1000) and cdouble is not null) (type: boolean) Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: ++++++++++++++ - keys: (cdecimal1 + cdecimal2) (type: decimal(11,3)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(11,3)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(21,11)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(23,9)), (cdecimal1 % 10) (type: decimal(5,3)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - top n: 10 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: ConvertDecimal64ToDecimal(col 7:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 6:decimal(8,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 6:decimal(8,3)/DECIMAL_64) -> 7:decimal(11,3)/DECIMAL_64) -> 37:decimal(11,3), ConvertDecimal64ToDecimal(col 10:decimal(11,3)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 9:decimal(10,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 8:decimal(9,2)/DECIMAL_64, decimal64Val 10, decimalVal 10)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 8:decimal(9,2)/DECIMAL_64) -> 9:decimal(10,3)/DECIMAL_64) -> 10:decimal(11,3)/DECIMAL_64) -> 38:decimal(11,3), DecimalColDivideDecimalColumn(col 39:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 39:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 40:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 40:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp + Select Operator + expressions: (cdecimal1 + cdecimal2) (type: decimal(11,3)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(11,3)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(21,11)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(23,9)), (cdecimal1 % 10) (type: decimal(5,3)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cdecimal1 + cdecimal2) (type: decimal(11,3)), (cdecimal1 - (2 * cdecimal2)) (type: decimal(11,3)), ((cdecimal1 + 2.34) / cdecimal2) (type: decimal(21,11)), (cdecimal1 * (cdecimal2 / 3.4)) (type: decimal(23,9)), (cdecimal1 % 10) (type: decimal(5,3)), UDFToInteger(cdecimal1) (type: int), UDFToShort(cdecimal2) (type: smallint), UDFToByte(cdecimal2) (type: tinyint), UDFToLong(cdecimal1) (type: bigint), UDFToBoolean(cdecimal1) (type: boolean), UDFToDouble(cdecimal2) (type: double), UDFToFloat(cdecimal1) (type: float), CAST( cdecimal2 AS STRING) (type: string), CAST( cdecimal1 AS TIMESTAMP) (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [7, 10, 13, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36] + selectExpressions: Decimal64ColAddDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 6:decimal(8,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 6:decimal(8,3)/DECIMAL_64) -> 7:decimal(11,3)/DECIMAL_64, Decimal64ColSubtractDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 9:decimal(10,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 8:decimal(9,2)/DECIMAL_64, decimal64Val 10, decimalVal 10)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 8:decimal(9,2)/DECIMAL_64) -> 9:decimal(10,3)/DECIMAL_64) -> 10:decimal(11,3)/DECIMAL_64, DecimalColDivideDecimalColumn(col 37:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 11:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 11:decimal(11,3)/DECIMAL_64) -> 37:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 13:decimal(21,11), DecimalColMultiplyDecimalColumn(col 14:decimal(10,3), col 38:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 14:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 38:decimal(12,6)) -> 16:decimal(23,9), DecimalColModuloDecimalScalar(col 17:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 17:decimal(10,3)) -> 18:decimal(5,3), CastDecimalToLong(col 19:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 19:decimal(10,3)) -> 20:int, CastDecimalToLong(col 21:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 21:decimal(7,2)) -> 22:smallint, CastDecimalToLong(col 23:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 23:decimal(7,2)) -> 24:tinyint, CastDecimalToLong(col 25:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 25:decimal(10,3)) -> 26:bigint, CastDecimalToBoolean(col 27:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 27:decimal(10,3)) -> 28:boolean, CastDecimalToDouble(col 29:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 29:decimal(7,2)) -> 30:double, CastDecimalToFloat(col 31:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 31:decimal(10,3)) -> 32:float, CastDecimalToString(col 33:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 33:decimal(7,2)) -> 34:string, CastDecimalToTimestamp(col 35:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 35:decimal(10,3)) -> 36:timestamp + Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 7:decimal(11,3), 10:decimal(11,3), 13:decimal(21,11), 16:decimal(23,9), 18:decimal(5,3), 20:int, 22:smallint, 24:tinyint, 26:bigint, 28:boolean, 30:double, 32:float, 34:string, 36:timestamp native: true - projectedOutputColumnNums: [11, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57] - selectExpressions: Decimal64ColAddDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 41:decimal(8,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 10, decimalVal 10) -> 41:decimal(8,3)/DECIMAL_64) -> 11:decimal(11,3)/DECIMAL_64, Decimal64ColSubtractDecimal64Column(col 1:decimal(10,3)/DECIMAL_64, col 43:decimal(10,3)/DECIMAL_64)(children: Decimal64ColScaleUp(col 42:decimal(9,2)/DECIMAL_64, decimal64Val 10, decimalVal 10)(children: Decimal64ScalarMultiplyDecimal64ColumnUnscaled(decimal64Val 2, decimalVal 2, col 2:decimal(7,2)/DECIMAL_64) -> 42:decimal(9,2)/DECIMAL_64) -> 43:decimal(10,3)/DECIMAL_64) -> 44:decimal(11,3)/DECIMAL_64, DecimalColDivideDecimalColumn(col 58:decimal(11,3), col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 45:decimal(11,3)/DECIMAL_64)(children: Decimal64ColAddDecimal64Scalar(col 1:decimal(10,3)/DECIMAL_64, decimal64Val 2340, decimalVal 2.34) -> 45:decimal(11,3)/DECIMAL_64) -> 58:decimal(11,3), ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 46:decimal(21,11), DecimalColMultiplyDecimalColumn(col 5:decimal(10,3), col 59:decimal(12,6))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3), ConvertDecimal64ToDecimal(col 15:decimal(12,6)/DECIMAL_64)(children: Decimal64ColDivideDecimal64Scalar(col 2:decimal(7,2)/DECIMAL_64, decimal64Val 340, decimalVal 3.4) -> 15:decimal(12,6)/DECIMAL_64) -> 59:decimal(12,6)) -> 47:decimal(23,9), DecimalColModuloDecimalScalar(col 5:decimal(10,3), val 10)(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 48:decimal(5,3), CastDecimalToLong(col 5:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 49:int, CastDecimalToLong(col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 50:smallint, CastDecimalToLong(col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 51:tinyint, CastDecimalToLong(col 5:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 52:bigint, CastDecimalToBoolean(col 5:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 53:boolean, CastDecimalToDouble(col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 54:double, CastDecimalToFloat(col 5:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 55:float, CastDecimalToString(col 12:decimal(7,2))(children: ConvertDecimal64ToDecimal(col 2:decimal(7,2)/DECIMAL_64) -> 12:decimal(7,2)) -> 56:string, CastDecimalToTimestamp(col 5:decimal(10,3))(children: ConvertDecimal64ToDecimal(col 1:decimal(10,3)/DECIMAL_64) -> 5:decimal(10,3)) -> 57:timestamp + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: decimal(11,3)), _col1 (type: decimal(11,3)), _col2 (type: decimal(21,11)), _col3 (type: decimal(23,9)), _col4 (type: decimal(5,3)), _col5 (type: int), _col6 (type: smallint), _col7 (type: tinyint), _col8 (type: bigint), _col9 (type: boolean), _col10 (type: double), _col11 (type: float), _col12 (type: string), _col13 (type: timestamp) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 11:decimal(11,3), 44:decimal(11,3), 46:decimal(21,11), 47:decimal(23,9), 48:decimal(5,3), 49:int, 50:smallint, 51:tinyint, 52:bigint, 53:boolean, 54:double, 55:float, 56:string, 57:timestamp - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1 Data size: 220 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -306,7 +288,7 @@ STAGE PLANS: includeColumns: [0, 1, 2] dataColumns: cdouble:double, cdecimal1:decimal(10,3)/DECIMAL_64, cdecimal2:decimal(7,2)/DECIMAL_64 partitionColumnCount: 0 - scratchColumnTypeNames: [decimal(10,3), decimal(8,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(11,3), decimal(11,3), decimal(12,6), decimal(8,3)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(21,11), decimal(23,9), decimal(5,3), bigint, bigint, bigint, bigint, bigint, double, double, string, timestamp, decimal(11,3), decimal(12,6)] + scratchColumnTypeNames: [decimal(10,3), decimal(8,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(11,3)/DECIMAL_64, decimal(7,2), decimal(21,11), decimal(10,3), decimal(12,6)/DECIMAL_64, decimal(23,9), decimal(10,3), decimal(5,3), decimal(10,3), bigint, decimal(7,2), bigint, decimal(7,2), bigint, decimal(10,3), bigint, decimal(10,3), bigint, decimal(7,2), double, decimal(10,3), double, decimal(7,2), string, decimal(10,3), timestamp, decimal(11,3), decimal(12,6)] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out index ff87190d0caf..9746a3c942c6 100644 --- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out @@ -278,34 +278,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_date (type: date) - null sort order: zz - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 5:int, col 2:date + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -332,34 +323,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - null sort order: zz - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:date + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 1, 4, 5, 0] + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 1, 4, 5, 0] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -1268,34 +1250,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_date (type: date) - null sort order: zz - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 4:int, col 5:date + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1322,34 +1295,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - null sort order: zz - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:date + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -2330,34 +2294,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_time (type: timestamp) - null sort order: zz - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 4:int, col 5:timestamp + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -2384,34 +2339,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - null sort order: zz - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:timestamp + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -2947,34 +2893,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_date (type: date) - null sort order: zz - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 5:int, col 2:date + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39456 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -3001,34 +2938,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - null sort order: zz - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:date + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: date), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 1, 4, 5, 0] + Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: int), _col2 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 1, 4, 5, 0] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: int), _col2 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7200 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: timestamp), _col4 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -3937,34 +3865,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_date (type: date) - null sort order: zz - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 4:int, col 5:date + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_time (type: timestamp), arr_delay (type: float), fl_num (type: int), fl_date (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -3991,34 +3910,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - null sort order: zz - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:date + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: timestamp), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: date) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: date) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: date) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: @@ -4999,34 +4909,25 @@ STAGE PLANS: Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++ - keys: fl_num (type: int), fl_time (type: timestamp) - null sort order: zz - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 4:int, col 5:timestamp + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: origin_city_name (type: string), dest_city_name (type: string), fl_date (type: date), arr_delay (type: float), fl_num (type: int), fl_time (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs (cache only) Map Vectorization: @@ -5053,34 +4954,25 @@ STAGE PLANS: className: VectorLimitOperator native: true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - null sort order: zz - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:timestamp + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: date), VALUE._col3 (type: float), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: timestamp) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: int), _col5 (type: timestamp) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col4 (type: int), _col5 (type: timestamp) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap Reduce Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out b/ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out index bc319e786fb4..00b01329e809 100644 --- a/ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out @@ -82,22 +82,17 @@ STAGE PLANS: TableScan alias: varchar_2_n0 Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: + - keys: key (type: varchar(10)) - null sort order: z + Select Operator + expressions: key (type: varchar(10)), value (type: varchar(20)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - top n: 5 - Select Operator - expressions: key (type: varchar(10)), value (type: varchar(20)) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: varchar(10)) + null sort order: z + sort order: + Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: varchar(10)) - null sort order: z - sort order: + - Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: varchar(20)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: varchar(20)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -212,22 +207,17 @@ STAGE PLANS: TableScan alias: varchar_2_n0 Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - Top N Key Operator - sort order: - - keys: key (type: varchar(10)) - null sort order: a + Select Operator + expressions: key (type: varchar(10)), value (type: varchar(20)) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - top n: 5 - Select Operator - expressions: key (type: varchar(10)), value (type: varchar(20)) - outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: varchar(10)) + null sort order: a + sort order: - Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: varchar(10)) - null sort order: a - sort order: - - Statistics: Num rows: 500 Data size: 94248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: varchar(20)) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: varchar(20)) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vectorization_7.q.out b/ql/src/test/results/clientpositive/llap/vectorization_7.q.out index b77998fc1cc0..0d51ca25bdcd 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_7.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_7.q.out @@ -88,35 +88,26 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 14:double, val -28815.0)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 15:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 15:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0)) predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28815.0D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++ - keys: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 10:boolean, col 3:bigint, col 1:smallint, col 0:tinyint, col 8:timestamp, col 6:string, LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Select Operator + expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 16, 17, 18, 19, 21, 23, 24, 25, 27] + selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) + null sort order: zzzzzzzzzzzzzzz + sort order: +++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 10:boolean, 3:bigint, 1:smallint, 0:tinyint, 8:timestamp, 6:string, 16:bigint, 17:int, 18:smallint, 19:tinyint, 21:int, 23:bigint, 24:int, 25:tinyint, 27:tinyint native: true - projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 20, 22, 26, 28, 30, 32, 33, 34, 36] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 20:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 22:int, LongColUnaryMinus(col 1:smallint) -> 26:smallint, LongColUnaryMinus(col 0:tinyint) -> 28:tinyint, LongColAddLongScalar(col 29:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 29:tinyint) -> 30:int, LongColMultiplyLongColumn(col 3:bigint, col 31:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 31:smallint) -> 32:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 33:int, LongColUnaryMinus(col 0:tinyint) -> 34:tinyint, LongColModuloLongColumn(col 35:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 35:tinyint) -> 36:tinyint - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - sort order: +++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 10:boolean, 3:bigint, 1:smallint, 0:tinyint, 8:timestamp, 6:string, 20:bigint, 22:int, 26:smallint, 28:tinyint, 30:int, 32:bigint, 33:int, 34:tinyint, 36:tinyint - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -133,7 +124,7 @@ STAGE PLANS: includeColumns: [0, 1, 2, 3, 5, 6, 7, 8, 9, 10] dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint] + scratchColumnTypeNames: [double, double, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint, bigint] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -157,19 +148,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14] - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -351,34 +342,25 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 14:double, val -28792.315)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 15:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 15:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0)) predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28792.315D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean) Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++ - keys: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - Statistics: Num rows: 11033 Data size: 2487724 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 10:boolean, col 3:bigint, col 1:smallint, col 0:tinyint, col 8:timestamp, col 6:string, LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Select Operator + expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 16, 17, 18, 19, 21, 23, 24, 25, 27] + selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 16:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 17:int, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColUnaryMinus(col 0:tinyint) -> 19:tinyint, LongColAddLongScalar(col 20:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 20:tinyint) -> 21:int, LongColMultiplyLongColumn(col 3:bigint, col 22:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 24:int, LongColUnaryMinus(col 0:tinyint) -> 25:tinyint, LongColModuloLongColumn(col 26:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 26:tinyint) -> 27:tinyint + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) + null sort order: zzzzzzzzzzzzzzz + sort order: +++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 20, 22, 26, 28, 30, 32, 33, 34, 36] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 20:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 22:int, LongColUnaryMinus(col 1:smallint) -> 26:smallint, LongColUnaryMinus(col 0:tinyint) -> 28:tinyint, LongColAddLongScalar(col 29:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 29:tinyint) -> 30:int, LongColMultiplyLongColumn(col 3:bigint, col 31:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 31:smallint) -> 32:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 33:int, LongColUnaryMinus(col 0:tinyint) -> 34:tinyint, LongColModuloLongColumn(col 35:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 35:tinyint) -> 36:tinyint - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint) - null sort order: zzzzzzzzzzzzzzz - sort order: +++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -406,19 +388,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14] - Statistics: Num rows: 11033 Data size: 1323416 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11033 Data size: 1754052 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 3102 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 4180 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vectorization_8.q.out b/ql/src/test/results/clientpositive/llap/vectorization_8.q.out index 7614c3d02257..b530183bfcd0 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_8.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_8.q.out @@ -84,35 +84,26 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterDoubleColNotEqualDoubleScalar(col 15:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 15:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean))) predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean) Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++++++++++++++ - keys: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 8:timestamp, col 5:double, col 10:boolean, col 6:string, col 4:float, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Select Operator + expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 5, 10, 6, 4, 16, 17, 18, 20, 23, 24, 25, 26, 30] + selectExpressions: DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 8:timestamp, 5:double, 10:boolean, 6:string, 4:float, 16:double, 17:double, 18:double, 20:float, 23:double, 24:double, 25:float, 26:float, 30:double native: true - projectedOutputColumnNums: [8, 5, 10, 6, 4, 14, 15, 19, 22, 28, 21, 27, 29, 34] - selectExpressions: DoubleColUnaryMinus(col 5:double) -> 14:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 15:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 19:double, DoubleColAddDoubleColumn(col 21:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 21:float) -> 22:float, DoubleColAddDoubleColumn(col 21:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 5:double) -> 21:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 27:float, DoubleColUnaryMinus(col 4:float) -> 29:float, DoubleColAddDoubleColumn(col 31:double, col 33:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 31:double, DoubleColAddDoubleColumn(col 32:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 32:float) -> 33:float) -> 34:double - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 8:timestamp, 5:double, 10:boolean, 6:string, 4:float, 14:double, 15:double, 19:double, 22:float, 28:double, 21:double, 27:float, 29:float, 34:double - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -129,7 +120,7 @@ STAGE PLANS: includeColumns: [2, 3, 4, 5, 6, 7, 8, 9, 10] dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean partitionColumnCount: 0 - scratchColumnTypeNames: [double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double] + scratchColumnTypeNames: [double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double, double] Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -153,19 +144,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13] - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -334,34 +325,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterDoubleColNotEqualDoubleScalar(col 15:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 15:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean))) predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean) Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++++++++++++++ - keys: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - null sort order: zzzzzzzzzzzzzz - Statistics: Num rows: 3059 Data size: 680930 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 8:timestamp, col 5:double, col 10:boolean, col 6:string, col 4:float, DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Select Operator + expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 5, 10, 6, 4, 16, 17, 18, 20, 23, 24, 25, 26, 30] + selectExpressions: DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 18:double, DoubleColAddDoubleColumn(col 19:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 19:float) -> 20:float, DoubleColAddDoubleColumn(col 21:double, col 22:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 5:double) -> 24:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 25:float, DoubleColUnaryMinus(col 4:float) -> 26:float, DoubleColAddDoubleColumn(col 27:double, col 29:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 27:double, DoubleColAddDoubleColumn(col 28:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 28:float) -> 29:float) -> 30:double + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) + null sort order: zzzzzzzzzzzzzz + sort order: ++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [8, 5, 10, 6, 4, 14, 15, 19, 22, 28, 21, 27, 29, 34] - selectExpressions: DoubleColUnaryMinus(col 5:double) -> 14:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 15:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 19:double, DoubleColAddDoubleColumn(col 21:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 21:float) -> 22:float, DoubleColAddDoubleColumn(col 21:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 21:double, CastLongToDouble(col 3:bigint) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 5:double) -> 21:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 27:float, DoubleColUnaryMinus(col 4:float) -> 29:float, DoubleColAddDoubleColumn(col 31:double, col 33:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 31:double, DoubleColAddDoubleColumn(col 32:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 32:float) -> 33:float) -> 34:double - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double) - null sort order: zzzzzzzzzzzzzz - sort order: ++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -389,19 +371,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13] - Statistics: Num rows: 3059 Data size: 410040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3059 Data size: 526290 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 2850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 3600 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out index f6eae5d9ca01..a3a6c54cab06 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_div0.q.out @@ -34,35 +34,26 @@ STAGE PLANS: Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true - Top N Key Operator - sort order: ++++ - keys: cint (type: int), ctinyint (type: tinyint), cbigint (type: bigint), cdouble (type: double) - null sort order: zzzz - Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 2:int, col 0:tinyint, col 3:bigint, col 5:double + Select Operator + expressions: cint (type: int), (UDFToDouble(cint) / 0.0D) (type: double), ctinyint (type: tinyint), (UDFToDouble(ctinyint) / 0.0D) (type: double), cbigint (type: bigint), (UDFToDouble(cbigint) / 0.0D) (type: double), cdouble (type: double), (cdouble / 0.0D) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), (UDFToDouble(cint) / 0.0D) (type: double), ctinyint (type: tinyint), (UDFToDouble(ctinyint) / 0.0D) (type: double), cbigint (type: bigint), (UDFToDouble(cbigint) / 0.0D) (type: double), cdouble (type: double), (cdouble / 0.0D) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 15, 0, 17, 3, 19, 5, 20] + selectExpressions: DoubleColDivideDoubleScalar(col 14:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 14:double) -> 15:double, DoubleColDivideDoubleScalar(col 16:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 16:double) -> 17:double, DoubleColDivideDoubleScalar(col 18:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 18:double) -> 19:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 20:double + Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col2 (type: tinyint), _col4 (type: bigint), _col6 (type: double) + null sort order: zzzz + sort order: ++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 15, 0, 17, 3, 19, 5, 20] - selectExpressions: DoubleColDivideDoubleScalar(col 14:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 14:double) -> 15:double, DoubleColDivideDoubleScalar(col 16:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 16:double) -> 17:double, DoubleColDivideDoubleScalar(col 18:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 18:double) -> 19:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 20:double - Statistics: Num rows: 12288 Data size: 538648 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col2 (type: tinyint), _col4 (type: bigint), _col6 (type: double) - null sort order: zzzz - sort order: ++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 12288 Data size: 538648 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: double), _col3 (type: double), _col5 (type: double), _col7 (type: double) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: double), _col3 (type: double), _col5 (type: double), _col7 (type: double) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -90,19 +81,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 4, 1, 5, 2, 6, 3, 7] - Statistics: Num rows: 12288 Data size: 538648 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 613400 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 5024 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 5024 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -269,34 +260,25 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterLongColGreaterLongScalar(col 3:bigint, val 0), FilterLongColLessLongScalar(col 3:bigint, val 100000000)) predicate: ((cbigint > 0L) and (cbigint < 100000000L)) (type: boolean) Statistics: Num rows: 3215 Data size: 38416 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++ - keys: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21)) - null sort order: zzz - Statistics: Num rows: 3215 Data size: 38416 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 14:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint) -> 16:double) -> 17:double, DecimalScalarDivideDecimalColumn(val 1.2, col 19:decimal(19,0))(children: CastLongToDecimal(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 19:decimal(19,0)) -> 20:decimal(22,21) + Select Operator + expressions: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21)) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21)) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [14, 17, 20] + selectExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 14:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint) -> 16:double) -> 17:double, DecimalScalarDivideDecimalColumn(val 1.2, col 19:decimal(19,0))(children: CastLongToDecimal(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 19:decimal(19,0)) -> 20:decimal(22,21) + Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: double), _col2 (type: decimal(22,21)) + null sort order: zzz + sort order: +++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [15, 21, 22] - selectExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 15:bigint, DoubleColDivideDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 16:double) -> 21:double, DecimalScalarDivideDecimalColumn(val 1.2, col 19:decimal(19,0))(children: CastLongToDecimal(col 18:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 18:bigint) -> 19:decimal(19,0)) -> 22:decimal(22,21) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: double), _col2 (type: decimal(22,21)) - null sort order: zzz - sort order: +++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3215 Data size: 411520 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -503,34 +485,25 @@ STAGE PLANS: predicateExpression: FilterExprAndExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -500.0), FilterDoubleColLessDoubleScalar(col 5:double, val -199.0)) predicate: ((cdouble >= -500.0D) and (cdouble < -199.0D)) (type: boolean) Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++++++ - keys: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double) - null sort order: zzzzzz - Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 14:double, DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 17:double, DoubleColDivideDoubleColumn(col 18:double, col 19:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 19:double) -> 20:double, DoubleColDivideDoubleColumn(col 21:double, col 22:double)(children: CastLongToDouble(col 3:bigint) -> 21:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 22:double) -> 23:double, DoubleScalarDivideDoubleColumn(val 3.0, col 24:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 24:double) -> 25:double, DoubleScalarDivideDoubleColumn(val 1.2, col 26:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 26:double) -> 27:double + Select Operator + expressions: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [14, 17, 20, 23, 25, 27] + selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 14:double, DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 17:double, DoubleColDivideDoubleColumn(col 18:double, col 19:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 19:double) -> 20:double, DoubleColDivideDoubleColumn(col 21:double, col 22:double)(children: CastLongToDouble(col 3:bigint) -> 21:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 22:double) -> 23:double, DoubleScalarDivideDoubleColumn(val 3.0, col 24:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 24:double) -> 25:double, DoubleScalarDivideDoubleColumn(val 1.2, col 26:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 26:double) -> 27:double + Statistics: Num rows: 20 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double) + null sort order: zzzzzz + sort order: ++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [15, 19, 21, 22, 18, 24] - selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 15:double, DoubleColDivideDoubleColumn(col 16:double, col 18:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double) -> 19:double, DoubleColDivideDoubleColumn(col 16:double, col 18:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double) -> 21:double, DoubleColDivideDoubleColumn(col 16:double, col 18:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double) -> 22:double, DoubleScalarDivideDoubleColumn(val 3.0, col 16:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 18:double, DoubleScalarDivideDoubleColumn(val 1.2, col 16:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 16:double) -> 24:double + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 20 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double), _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: double), _col5 (type: double) - null sort order: zzzzzz - sort order: ++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 20 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -737,34 +710,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2:int, val 500000000), FilterDoubleColGreaterDoubleScalar(col 5:double, val 1.0E9), FilterLongColEqualLongScalar(col 0:tinyint, val 0)) predicate: ((cint > 500000000) or (cdouble > 1.0E9D) or (ctinyint = 0Y)) (type: boolean) Statistics: Num rows: 3380 Data size: 60576 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++ - keys: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (UDFToDouble(cint) / UDFToDouble((cint - 528534767))) (type: double), (UDFToDouble(cbigint) / UDFToDouble((cbigint - 1018195815L))) (type: double), (UDFToDouble(ctinyint) / UDFToDouble(ctinyint)) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815L)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) - null sort order: zzzzzzzzz - Statistics: Num rows: 3380 Data size: 60576 Basic stats: COMPLETE Column stats: COMPLETE - top n: 100 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 2:int, col 3:bigint, col 0:tinyint, DoubleColDivideDoubleColumn(col 14:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 15:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 15:int) -> 16:double) -> 17:double, DoubleColDivideDoubleColumn(col 18:double, col 20:double)(children: CastLongToDouble(col 3:bigint) -> 18:double, CastLongToDouble(col 19:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 19:bigint) -> 20:double) -> 21:double, DoubleColDivideDoubleColumn(col 22:double, col 23:double)(children: CastLongToDouble(col 0:tinyint) -> 22:double, CastLongToDouble(col 0:tinyint) -> 23:double) -> 24:double, LongColModuloLongColumn(col 2:int, col 25:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 25:int) -> 26:int, LongColModuloLongColumn(col 3:bigint, col 27:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 27:bigint) -> 28:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 29:tinyint + Select Operator + expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (UDFToDouble(cint) / UDFToDouble((cint - 528534767))) (type: double), (UDFToDouble(cbigint) / UDFToDouble((cbigint - 1018195815L))) (type: double), (UDFToDouble(ctinyint) / UDFToDouble(ctinyint)) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815L)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (UDFToDouble(cint) / UDFToDouble((cint - 528534767))) (type: double), (UDFToDouble(cbigint) / UDFToDouble((cbigint - 1018195815L))) (type: double), (UDFToDouble(ctinyint) / UDFToDouble(ctinyint)) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815L)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 0, 17, 21, 24, 26, 28, 29] + selectExpressions: DoubleColDivideDoubleColumn(col 14:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 15:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 15:int) -> 16:double) -> 17:double, DoubleColDivideDoubleColumn(col 18:double, col 20:double)(children: CastLongToDouble(col 3:bigint) -> 18:double, CastLongToDouble(col 19:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 19:bigint) -> 20:double) -> 21:double, DoubleColDivideDoubleColumn(col 22:double, col 23:double)(children: CastLongToDouble(col 0:tinyint) -> 22:double, CastLongToDouble(col 0:tinyint) -> 23:double) -> 24:double, LongColModuloLongColumn(col 2:int, col 25:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 25:int) -> 26:int, LongColModuloLongColumn(col 3:bigint, col 27:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 27:bigint) -> 28:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 29:tinyint + Statistics: Num rows: 3380 Data size: 175584 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint) + null sort order: zzzzzzzzz + sort order: +++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 0, 18, 20, 22, 19, 25, 15] - selectExpressions: DoubleColDivideDoubleColumn(col 14:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 15:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 15:int) -> 16:double) -> 18:double, DoubleColDivideDoubleColumn(col 14:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 14:double, CastLongToDouble(col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 15:bigint) -> 16:double) -> 20:double, DoubleColDivideDoubleColumn(col 14:double, col 16:double)(children: CastLongToDouble(col 0:tinyint) -> 14:double, CastLongToDouble(col 0:tinyint) -> 16:double) -> 22:double, LongColModuloLongColumn(col 2:int, col 15:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 15:int) -> 19:int, LongColModuloLongColumn(col 3:bigint, col 15:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 15:bigint) -> 25:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 15:tinyint - Statistics: Num rows: 3380 Data size: 161872 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col3 (type: double), _col4 (type: double), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint) - null sort order: zzzzzzzzz - sort order: +++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3380 Data size: 161872 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3380 Data size: 175584 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -792,19 +756,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8] - Statistics: Num rows: 3380 Data size: 161872 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3380 Data size: 175584 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 100 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 100 Data size: 4800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out index e626a7c760d2..01bc54962311 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_limit.q.out @@ -34,21 +34,16 @@ STAGE PLANS: Filter Operator predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean) Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: cbigint (type: bigint), cdouble (type: double) - null sort order: zz - Statistics: Num rows: 2048 Data size: 30600 Basic stats: COMPLETE Column stats: COMPLETE - top n: 7 - Select Operator - expressions: cbigint (type: bigint), cdouble (type: double) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint), _col1 (type: double) - null sort order: zz - sort order: ++ - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cbigint (type: bigint), cdouble (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col1 (type: double) + null sort order: zz + sort order: ++ + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -72,13 +67,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: double) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2048 Data size: 16176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2048 Data size: 24480 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 7 - Statistics: Num rows: 7 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 7 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -148,34 +143,25 @@ STAGE PLANS: predicateExpression: SelectColumnIsNotNull(col 0:tinyint) predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++ - keys: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - null sort order: zzz - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - top n: 20 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:tinyint, col 5:double, col 1:smallint + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 5, 1] + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double), _col2 (type: smallint) + null sort order: zzz + sort order: +++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: 0:tinyint, 5:double, 1:smallint native: true - projectedOutputColumnNums: [0, 5, 1] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double), _col2 (type: smallint) - null sort order: zzz - sort order: +++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 0:tinyint, 5:double, 1:smallint - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.3 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -216,19 +202,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 20 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 20 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 20 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 256 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out b/ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out index e15c7cb047d7..273e8738415d 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_offset_limit.q.out @@ -139,34 +139,25 @@ STAGE PLANS: predicateExpression: SelectColumnIsNotNull(col 0:tinyint) predicate: ctinyint is not null (type: boolean) Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: ctinyint (type: tinyint), cdouble (type: double) - null sort order: zz - Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE - top n: 13 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:tinyint, col 5:double + Select Operator + expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint) - outputColumnNames: _col0, _col1, _col2 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 5, 1] + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: double) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 5, 1] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint), _col1 (type: double) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: smallint) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: smallint) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -194,20 +185,20 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2] - Statistics: Num rows: 9173 Data size: 72384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9173 Data size: 109584 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 3 Limit Vectorization: className: VectorLimitOperator native: true Offset of rows: 10 - Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out b/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out index f2ce2c2b5b93..443b732ee358 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out @@ -79,21 +79,16 @@ STAGE PLANS: TableScan alias: alltypesorc_part Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: (cdouble + 2.0D) (type: double) - null sort order: z + Select Operator + expressions: (cdouble + 2.0D) (type: double) + outputColumnNames: _col0 Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Select Operator - expressions: (cdouble + 2.0D) (type: double) - outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: double) + null sort order: z + sort order: + Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: double) - null sort order: z - sort order: + - Statistics: Num rows: 200 Data size: 1600 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index da82903d7963..07ff00b0170b 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -1134,34 +1134,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 5:double, col 14:double)(children: CastLongToDouble(col 0:tinyint) -> 14:double), FilterDoubleColGreaterEqualDoubleColumn(col 4:float, col 15:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 15:float)), FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 11:boolean, val 1), FilterDecimal64ColLessDecimal64Scalar(col 16:decimal(8,3)/DECIMAL_64, val 79553)(children: CastLongToDecimal64(col 1:smallint) -> 16:decimal(8,3)/DECIMAL_64), FilterLongColNotEqualLongScalar(col 0:int, val -257)(children: col 0:tinyint)), FilterExprAndExpr(children: FilterLongColLessLongColumn(col 2:bigint, col 3:bigint)(children: col 2:int), FilterLongColGreaterLongColumn(col 0:bigint, col 3:bigint)(children: col 0:tinyint)), FilterExprAndExpr(children: FilterStringColRegExpStringScalar(col 6:string, pattern a.*), FilterStringColLikeStringScalar(col 7:string, pattern %ss%))) predicate: (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((cboolean2 <> 1) and (CAST( csmallint AS decimal(8,3)) < 79.553) and (UDFToInteger(ctinyint) <> -257)) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint)) or (cstring1 regexp 'a.*' and (cstring2 like '%ss%'))) (type: boolean) Statistics: Num rows: 9898 Data size: 2261486 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++++++++++ - keys: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728L * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - CAST( cint AS decimal(10,0))) (type: decimal(14,3)), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - CAST( cint AS decimal(10,0))) - -26.28) (type: decimal(15,3)), (- cfloat) (type: float), (cdouble * -89010.0D) (type: double), (UDFToDouble(ctinyint) / 988888.0D) (type: double), (- ctinyint) (type: tinyint), (79.553 / CAST( ctinyint AS decimal(3,0))) (type: decimal(9,7)) - null sort order: zzzzzzzzzzzzzzzzzzzzzzz - Statistics: Num rows: 9898 Data size: 2261486 Basic stats: COMPLETE Column stats: COMPLETE - top n: 50 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 2:int, col 5:double, col 9:timestamp, col 6:string, col 11:boolean, col 0:tinyint, col 4:float, col 8:timestamp, col 1:smallint, col 3:bigint, LongScalarMultiplyLongColumn(val -3728, col 3:bigint) -> 17:bigint, LongColUnaryMinus(col 2:int) -> 18:int, DecimalScalarSubtractDecimalColumn(val -863.257, col 19:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 19:decimal(10,0)) -> 20:decimal(14,3), LongColUnaryMinus(col 1:smallint) -> 21:smallint, LongColSubtractLongColumn(col 1:smallint, col 22:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:smallint, LongColAddLongColumn(col 25:smallint, col 26:smallint)(children: LongColSubtractLongColumn(col 1:smallint, col 24:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 24:smallint) -> 25:smallint, LongColUnaryMinus(col 1:smallint) -> 26:smallint) -> 27:smallint, DoubleColDivideDoubleColumn(col 28:double, col 29:double)(children: CastLongToDouble(col 2:int) -> 28:double, CastLongToDouble(col 2:int) -> 29:double) -> 30:double, DecimalColSubtractDecimalScalar(col 34:decimal(14,3), val -26.28)(children: DecimalScalarSubtractDecimalColumn(val -863.257, col 33:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 33:decimal(10,0)) -> 34:decimal(14,3)) -> 35:decimal(15,3), DoubleColUnaryMinus(col 4:float) -> 36:float, DoubleColMultiplyDoubleScalar(col 5:double, val -89010.0) -> 37:double, DoubleColDivideDoubleScalar(col 38:double, val 988888.0)(children: CastLongToDouble(col 0:tinyint) -> 38:double) -> 39:double, LongColUnaryMinus(col 0:tinyint) -> 40:tinyint, DecimalScalarDivideDecimalColumn(val 79.553, col 41:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 41:decimal(3,0)) -> 42:decimal(9,7) + Select Operator + expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728L * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - CAST( cint AS decimal(10,0))) (type: decimal(14,3)), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - CAST( cint AS decimal(10,0))) - -26.28) (type: decimal(15,3)), (- cfloat) (type: float), (cdouble * -89010.0D) (type: double), (UDFToDouble(ctinyint) / 988888.0D) (type: double), (- ctinyint) (type: tinyint), (79.553 / CAST( ctinyint AS decimal(3,0))) (type: decimal(9,7)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728L * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - CAST( cint AS decimal(10,0))) (type: decimal(14,3)), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - CAST( cint AS decimal(10,0))) - -26.28) (type: decimal(15,3)), (- cfloat) (type: float), (cdouble * -89010.0D) (type: double), (UDFToDouble(ctinyint) / 988888.0D) (type: double), (- ctinyint) (type: tinyint), (79.553 / CAST( ctinyint AS decimal(3,0))) (type: decimal(9,7)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 5, 9, 6, 11, 0, 4, 8, 1, 3, 17, 18, 20, 21, 23, 27, 30, 35, 36, 37, 39, 40, 42] + selectExpressions: LongScalarMultiplyLongColumn(val -3728, col 3:bigint) -> 17:bigint, LongColUnaryMinus(col 2:int) -> 18:int, DecimalScalarSubtractDecimalColumn(val -863.257, col 19:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 19:decimal(10,0)) -> 20:decimal(14,3), LongColUnaryMinus(col 1:smallint) -> 21:smallint, LongColSubtractLongColumn(col 1:smallint, col 22:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 22:smallint) -> 23:smallint, LongColAddLongColumn(col 25:smallint, col 26:smallint)(children: LongColSubtractLongColumn(col 1:smallint, col 24:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 24:smallint) -> 25:smallint, LongColUnaryMinus(col 1:smallint) -> 26:smallint) -> 27:smallint, DoubleColDivideDoubleColumn(col 28:double, col 29:double)(children: CastLongToDouble(col 2:int) -> 28:double, CastLongToDouble(col 2:int) -> 29:double) -> 30:double, DecimalColSubtractDecimalScalar(col 34:decimal(14,3), val -26.28)(children: DecimalScalarSubtractDecimalColumn(val -863.257, col 33:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 33:decimal(10,0)) -> 34:decimal(14,3)) -> 35:decimal(15,3), DoubleColUnaryMinus(col 4:float) -> 36:float, DoubleColMultiplyDoubleScalar(col 5:double, val -89010.0) -> 37:double, DoubleColDivideDoubleScalar(col 38:double, val 988888.0)(children: CastLongToDouble(col 0:tinyint) -> 38:double) -> 39:double, LongColUnaryMinus(col 0:tinyint) -> 40:tinyint, DecimalScalarDivideDecimalColumn(val 79.553, col 41:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 41:decimal(3,0)) -> 42:decimal(9,7) + Statistics: Num rows: 9898 Data size: 5432062 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: timestamp), _col3 (type: string), _col4 (type: boolean), _col5 (type: tinyint), _col6 (type: float), _col7 (type: timestamp), _col8 (type: smallint), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: int), _col12 (type: decimal(14,3)), _col13 (type: smallint), _col14 (type: smallint), _col15 (type: smallint), _col16 (type: double), _col17 (type: decimal(15,3)), _col18 (type: float), _col19 (type: double), _col20 (type: double), _col21 (type: tinyint), _col22 (type: decimal(9,7)) + null sort order: zzzzzzzzzzzzzzzzzzzzzzz + sort order: +++++++++++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 5, 9, 6, 11, 0, 4, 8, 1, 3, 22, 24, 34, 25, 43, 45, 28, 48, 14, 15, 38, 26, 49] - selectExpressions: LongScalarMultiplyLongColumn(val -3728, col 3:bigint) -> 22:bigint, LongColUnaryMinus(col 2:int) -> 24:int, DecimalScalarSubtractDecimalColumn(val -863.257, col 19:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 19:decimal(10,0)) -> 34:decimal(14,3), LongColUnaryMinus(col 1:smallint) -> 25:smallint, LongColSubtractLongColumn(col 1:smallint, col 26:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 26:smallint) -> 43:smallint, LongColAddLongColumn(col 44:smallint, col 26:smallint)(children: LongColSubtractLongColumn(col 1:smallint, col 26:smallint)(children: LongColUnaryMinus(col 1:smallint) -> 26:smallint) -> 44:smallint, LongColUnaryMinus(col 1:smallint) -> 26:smallint) -> 45:smallint, DoubleColDivideDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 2:int) -> 15:double) -> 28:double, DecimalColSubtractDecimalScalar(col 47:decimal(14,3), val -26.28)(children: DecimalScalarSubtractDecimalColumn(val -863.257, col 19:decimal(10,0))(children: CastLongToDecimal(col 2:int) -> 19:decimal(10,0)) -> 47:decimal(14,3)) -> 48:decimal(15,3), DoubleColUnaryMinus(col 4:float) -> 14:float, DoubleColMultiplyDoubleScalar(col 5:double, val -89010.0) -> 15:double, DoubleColDivideDoubleScalar(col 29:double, val 988888.0)(children: CastLongToDouble(col 0:tinyint) -> 29:double) -> 38:double, LongColUnaryMinus(col 0:tinyint) -> 26:tinyint, DecimalScalarDivideDecimalColumn(val 79.553, col 41:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 41:decimal(3,0)) -> 49:decimal(9,7) - Statistics: Num rows: 9898 Data size: 4905318 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: double), _col2 (type: timestamp), _col3 (type: string), _col4 (type: boolean), _col5 (type: tinyint), _col6 (type: float), _col7 (type: timestamp), _col8 (type: smallint), _col9 (type: bigint), _col10 (type: bigint), _col11 (type: int), _col12 (type: decimal(14,3)), _col13 (type: smallint), _col14 (type: smallint), _col15 (type: smallint), _col16 (type: double), _col17 (type: decimal(15,3)), _col18 (type: float), _col19 (type: double), _col20 (type: double), _col21 (type: tinyint), _col22 (type: decimal(9,7)) - null sort order: zzzzzzzzzzzzzzzzzzzzzzz - sort order: +++++++++++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 9898 Data size: 4905318 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 9898 Data size: 5432062 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1189,19 +1180,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - Statistics: Num rows: 9898 Data size: 4905318 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9898 Data size: 5432062 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 50 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 50 Data size: 25060 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 50 Data size: 27580 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 50 Data size: 25060 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 50 Data size: 27580 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1446,34 +1437,25 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterLongColLessLongScalar(col 10:boolean, val 0), FilterLongColEqualLongScalar(col 3:bigint, val 359), FilterExprAndExpr(children: FilterLongColLessLongScalar(col 0:int, val 197)(children: col 0:tinyint), FilterLongColEqualLongColumn(col 2:bigint, col 3:bigint)(children: col 2:int)), FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 6:string, pattern %ss), FilterDoubleColLessEqualDoubleColumn(col 4:float, col 14:float)(children: CastLongToFloatViaLongToDouble(col 0:tinyint) -> 14:float))) predicate: ((cboolean1 < 0) or (cbigint = 359L) or ((UDFToInteger(ctinyint) < 197) and (UDFToLong(cint) = cbigint)) or ((cstring1 like '%ss') and (cfloat <= UDFToFloat(ctinyint)))) (type: boolean) Statistics: Num rows: 8194 Data size: 1651860 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++++++++++++ - keys: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cboolean1 (type: boolean), cfloat (type: float), cdouble (type: double), ctimestamp2 (type: timestamp), csmallint (type: smallint), cstring2 (type: string), cboolean2 (type: boolean), (UDFToDouble(cint) / UDFToDouble(cbigint)) (type: double), (CAST( cbigint AS decimal(19,0)) % 79.553) (type: decimal(5,3)), (- (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (10.175 % cfloat) (type: float), (- cfloat) (type: float), (cfloat - (- cfloat)) (type: float), ((cfloat - (- cfloat)) % -6432.0) (type: float), (cdouble * UDFToDouble(csmallint)) (type: double), (- cdouble) (type: double), (- cbigint) (type: bigint), (UDFToDouble(cfloat) - (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (- csmallint) (type: smallint), (3569L % cbigint) (type: bigint), (359.0D - cdouble) (type: double), (- csmallint) (type: smallint) - null sort order: zzzzzzzzzzzzzzzzzzzzzzzzz - Statistics: Num rows: 8194 Data size: 1651860 Basic stats: COMPLETE Column stats: COMPLETE - top n: 25 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 2:int, col 3:bigint, col 6:string, col 10:boolean, col 4:float, col 5:double, col 9:timestamp, col 1:smallint, col 7:string, col 11:boolean, DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 15:double, CastLongToDouble(col 3:bigint) -> 16:double) -> 17:double, DecimalColModuloDecimalScalar(col 18:decimal(19,0), val 79.553)(children: CastLongToDecimal(col 3:bigint) -> 18:decimal(19,0)) -> 19:decimal(5,3), DoubleColUnaryMinus(col 22:double)(children: DoubleColDivideDoubleColumn(col 20:double, col 21:double)(children: CastLongToDouble(col 2:int) -> 20:double, CastLongToDouble(col 3:bigint) -> 21:double) -> 22:double) -> 23:double, DoubleScalarModuloDoubleColumn(val 10.175000190734863, col 4:float) -> 24:float, DoubleColUnaryMinus(col 4:float) -> 25:float, DoubleColSubtractDoubleColumn(col 4:float, col 26:float)(children: DoubleColUnaryMinus(col 4:float) -> 26:float) -> 27:float, DoubleColModuloDoubleScalar(col 29:float, val -6432.0)(children: DoubleColSubtractDoubleColumn(col 4:float, col 28:float)(children: DoubleColUnaryMinus(col 4:float) -> 28:float) -> 29:float) -> 30:float, DoubleColMultiplyDoubleColumn(col 5:double, col 31:double)(children: CastLongToDouble(col 1:smallint) -> 31:double) -> 32:double, DoubleColUnaryMinus(col 5:double) -> 33:double, LongColUnaryMinus(col 3:bigint) -> 34:bigint, DoubleColSubtractDoubleColumn(col 4:double, col 37:double)(children: col 4:float, DoubleColDivideDoubleColumn(col 35:double, col 36:double)(children: CastLongToDouble(col 2:int) -> 35:double, CastLongToDouble(col 3:bigint) -> 36:double) -> 37:double) -> 38:double, LongColUnaryMinus(col 1:smallint) -> 39:smallint, LongScalarModuloLongColumn(val 3569, col 3:bigint) -> 40:bigint, DoubleScalarSubtractDoubleColumn(val 359.0, col 5:double) -> 41:double, LongColUnaryMinus(col 1:smallint) -> 42:smallint + Select Operator + expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cboolean1 (type: boolean), cfloat (type: float), cdouble (type: double), ctimestamp2 (type: timestamp), csmallint (type: smallint), cstring2 (type: string), cboolean2 (type: boolean), (UDFToDouble(cint) / UDFToDouble(cbigint)) (type: double), (CAST( cbigint AS decimal(19,0)) % 79.553) (type: decimal(5,3)), (- (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (10.175 % cfloat) (type: float), (- cfloat) (type: float), (cfloat - (- cfloat)) (type: float), ((cfloat - (- cfloat)) % -6432.0) (type: float), (cdouble * UDFToDouble(csmallint)) (type: double), (- cdouble) (type: double), (- cbigint) (type: bigint), (UDFToDouble(cfloat) - (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (- csmallint) (type: smallint), (3569L % cbigint) (type: bigint), (359.0D - cdouble) (type: double), (- csmallint) (type: smallint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), cbigint (type: bigint), cstring1 (type: string), cboolean1 (type: boolean), cfloat (type: float), cdouble (type: double), ctimestamp2 (type: timestamp), csmallint (type: smallint), cstring2 (type: string), cboolean2 (type: boolean), (UDFToDouble(cint) / UDFToDouble(cbigint)) (type: double), (CAST( cbigint AS decimal(19,0)) % 79.553) (type: decimal(5,3)), (- (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (10.175 % cfloat) (type: float), (- cfloat) (type: float), (cfloat - (- cfloat)) (type: float), ((cfloat - (- cfloat)) % -6432.0) (type: float), (cdouble * UDFToDouble(csmallint)) (type: double), (- cdouble) (type: double), (- cbigint) (type: bigint), (UDFToDouble(cfloat) - (UDFToDouble(cint) / UDFToDouble(cbigint))) (type: double), (- csmallint) (type: smallint), (3569L % cbigint) (type: bigint), (359.0D - cdouble) (type: double), (- csmallint) (type: smallint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 3, 6, 10, 4, 5, 9, 1, 7, 11, 17, 19, 23, 24, 25, 27, 30, 32, 33, 34, 38, 39, 40, 41, 42] + selectExpressions: DoubleColDivideDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 15:double, CastLongToDouble(col 3:bigint) -> 16:double) -> 17:double, DecimalColModuloDecimalScalar(col 18:decimal(19,0), val 79.553)(children: CastLongToDecimal(col 3:bigint) -> 18:decimal(19,0)) -> 19:decimal(5,3), DoubleColUnaryMinus(col 22:double)(children: DoubleColDivideDoubleColumn(col 20:double, col 21:double)(children: CastLongToDouble(col 2:int) -> 20:double, CastLongToDouble(col 3:bigint) -> 21:double) -> 22:double) -> 23:double, DoubleScalarModuloDoubleColumn(val 10.175000190734863, col 4:float) -> 24:float, DoubleColUnaryMinus(col 4:float) -> 25:float, DoubleColSubtractDoubleColumn(col 4:float, col 26:float)(children: DoubleColUnaryMinus(col 4:float) -> 26:float) -> 27:float, DoubleColModuloDoubleScalar(col 29:float, val -6432.0)(children: DoubleColSubtractDoubleColumn(col 4:float, col 28:float)(children: DoubleColUnaryMinus(col 4:float) -> 28:float) -> 29:float) -> 30:float, DoubleColMultiplyDoubleColumn(col 5:double, col 31:double)(children: CastLongToDouble(col 1:smallint) -> 31:double) -> 32:double, DoubleColUnaryMinus(col 5:double) -> 33:double, LongColUnaryMinus(col 3:bigint) -> 34:bigint, DoubleColSubtractDoubleColumn(col 4:double, col 37:double)(children: col 4:float, DoubleColDivideDoubleColumn(col 35:double, col 36:double)(children: CastLongToDouble(col 2:int) -> 35:double, CastLongToDouble(col 3:bigint) -> 36:double) -> 37:double) -> 38:double, LongColUnaryMinus(col 1:smallint) -> 39:smallint, LongScalarModuloLongColumn(val 3569, col 3:bigint) -> 40:bigint, DoubleScalarSubtractDoubleColumn(val 359.0, col 5:double) -> 41:double, LongColUnaryMinus(col 1:smallint) -> 42:smallint + Statistics: Num rows: 8194 Data size: 3266188 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: boolean), _col4 (type: float), _col5 (type: double), _col6 (type: timestamp), _col7 (type: smallint), _col8 (type: string), _col9 (type: boolean), _col10 (type: double), _col11 (type: decimal(5,3)), _col12 (type: double), _col13 (type: float), _col14 (type: float), _col15 (type: float), _col16 (type: float), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: double), _col21 (type: smallint), _col22 (type: bigint), _col23 (type: double), _col24 (type: smallint) + null sort order: zzzzzzzzzzzzzzzzzzzzzzzzz + sort order: +++++++++++++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 3, 6, 10, 4, 5, 9, 1, 7, 11, 16, 43, 14, 15, 20, 22, 21, 28, 26, 44, 29, 45, 46, 31, 47] - selectExpressions: DoubleColDivideDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 3:bigint) -> 15:double) -> 16:double, DecimalColModuloDecimalScalar(col 18:decimal(19,0), val 79.553)(children: CastLongToDecimal(col 3:bigint) -> 18:decimal(19,0)) -> 43:decimal(5,3), DoubleColUnaryMinus(col 20:double)(children: DoubleColDivideDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 2:int) -> 14:double, CastLongToDouble(col 3:bigint) -> 15:double) -> 20:double) -> 14:double, DoubleScalarModuloDoubleColumn(val 10.175000190734863, col 4:float) -> 15:float, DoubleColUnaryMinus(col 4:float) -> 20:float, DoubleColSubtractDoubleColumn(col 4:float, col 21:float)(children: DoubleColUnaryMinus(col 4:float) -> 21:float) -> 22:float, DoubleColModuloDoubleScalar(col 26:float, val -6432.0)(children: DoubleColSubtractDoubleColumn(col 4:float, col 21:float)(children: DoubleColUnaryMinus(col 4:float) -> 21:float) -> 26:float) -> 21:float, DoubleColMultiplyDoubleColumn(col 5:double, col 26:double)(children: CastLongToDouble(col 1:smallint) -> 26:double) -> 28:double, DoubleColUnaryMinus(col 5:double) -> 26:double, LongColUnaryMinus(col 3:bigint) -> 44:bigint, DoubleColSubtractDoubleColumn(col 4:double, col 35:double)(children: col 4:float, DoubleColDivideDoubleColumn(col 29:double, col 31:double)(children: CastLongToDouble(col 2:int) -> 29:double, CastLongToDouble(col 3:bigint) -> 31:double) -> 35:double) -> 29:double, LongColUnaryMinus(col 1:smallint) -> 45:smallint, LongScalarModuloLongColumn(val 3569, col 3:bigint) -> 46:bigint, DoubleScalarSubtractDoubleColumn(val 359.0, col 5:double) -> 31:double, LongColUnaryMinus(col 1:smallint) -> 47:smallint - Statistics: Num rows: 8194 Data size: 2713706 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: boolean), _col4 (type: float), _col5 (type: double), _col6 (type: timestamp), _col7 (type: smallint), _col8 (type: string), _col9 (type: boolean), _col10 (type: double), _col11 (type: decimal(5,3)), _col12 (type: double), _col13 (type: float), _col14 (type: float), _col15 (type: float), _col16 (type: float), _col17 (type: double), _col18 (type: double), _col19 (type: bigint), _col20 (type: double), _col21 (type: smallint), _col22 (type: bigint), _col23 (type: double), _col24 (type: smallint) - null sort order: zzzzzzzzzzzzzzzzzzzzzzzzz - sort order: +++++++++++++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 8194 Data size: 2713706 Basic stats: COMPLETE Column stats: COMPLETE + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 8194 Data size: 3266188 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1501,19 +1483,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 21] - Statistics: Num rows: 8194 Data size: 2713706 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8194 Data size: 3266188 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 8458 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 10320 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 8458 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 10320 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1707,35 +1689,26 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleColumn(col 14:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 3:bigint) -> 14:float), FilterDecimal64ColGreaterEqualDecimal64Scalar(col 15:decimal(7,2)/DECIMAL_64, val -2628)(children: CastLongToDecimal64(col 1:smallint) -> 15:decimal(7,2)/DECIMAL_64)), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleColumn(col 5:double, col 16:double)(children: CastLongToDouble(col 3:bigint) -> 16:double), FilterStringGroupColGreaterEqualStringScalar(col 6:string, val ss), FilterDoubleColNotEqualDoubleColumn(col 17:double, col 5:double)(children: CastLongToDouble(col 2:int) -> 17:double)), FilterLongColEqualLongScalar(col 0:int, val -89010)(children: col 0:tinyint), FilterExprAndExpr(children: FilterDecimal64ColGreaterDecimal64Scalar(col 18:decimal(7,2)/DECIMAL_64, val -2628)(children: CastLongToDecimal64(col 1:smallint) -> 18:decimal(7,2)/DECIMAL_64), FilterStringColLikeStringScalar(col 7:string, pattern ss))) predicate: (((UDFToFloat(cbigint) <= cfloat) and (CAST( csmallint AS decimal(7,2)) >= -26.28)) or ((cdouble <= UDFToDouble(cbigint)) and (cstring1 >= 'ss') and (UDFToDouble(cint) <> cdouble)) or (UDFToInteger(ctinyint) = -89010) or ((CAST( csmallint AS decimal(7,2)) > -26.28) and (cstring2 like 'ss'))) (type: boolean) Statistics: Num rows: 10922 Data size: 2201730 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++++++++ - keys: cboolean1 (type: boolean), cstring1 (type: string), ctimestamp2 (type: timestamp), cfloat (type: float), cbigint (type: bigint), cdouble (type: double), cint (type: int), csmallint (type: smallint), (cint + UDFToInteger(csmallint)) (type: int), (cbigint - UDFToLong(ctinyint)) (type: bigint), (- cbigint) (type: bigint), (- cfloat) (type: float), ((cbigint - UDFToLong(ctinyint)) + cbigint) (type: bigint), (cdouble / cdouble) (type: double), (- cdouble) (type: double), (UDFToLong((cint + UDFToInteger(csmallint))) * (- cbigint)) (type: bigint), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (-1.389 / CAST( ctinyint AS decimal(3,0))) (type: decimal(8,7)), (UDFToDouble(cbigint) % cdouble) (type: double), (- csmallint) (type: smallint), (UDFToInteger(csmallint) + (cint + UDFToInteger(csmallint))) (type: int) - null sort order: zzzzzzzzzzzzzzzzzzzzz - Statistics: Num rows: 10922 Data size: 2201730 Basic stats: COMPLETE Column stats: COMPLETE - top n: 75 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 10:boolean, col 6:string, col 9:timestamp, col 4:float, col 3:bigint, col 5:double, col 2:int, col 1:smallint, LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 19:int, LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 20:bigint, LongColUnaryMinus(col 3:bigint) -> 21:bigint, DoubleColUnaryMinus(col 4:float) -> 22:float, LongColAddLongColumn(col 23:bigint, col 3:bigint)(children: LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 23:bigint) -> 24:bigint, DoubleColDivideDoubleColumn(col 5:double, col 5:double) -> 25:double, DoubleColUnaryMinus(col 5:double) -> 26:double, LongColMultiplyLongColumn(col 27:bigint, col 28:bigint)(children: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 27:int, LongColUnaryMinus(col 3:bigint) -> 28:bigint) -> 29:bigint, DoubleColAddDoubleColumn(col 30:double, col 31:double)(children: DoubleColUnaryMinus(col 5:double) -> 30:double, CastLongToDouble(col 3:bigint) -> 31:double) -> 32:double, DecimalScalarDivideDecimalColumn(val -1.389, col 33:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 33:decimal(3,0)) -> 34:decimal(8,7), DoubleColModuloDoubleColumn(col 35:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 35:double) -> 36:double, LongColUnaryMinus(col 1:smallint) -> 37:smallint, LongColAddLongColumn(col 1:int, col 38:int)(children: col 1:smallint, LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 38:int) -> 39:int + Select Operator + expressions: cint (type: int), cstring1 (type: string), cboolean2 (type: boolean), ctimestamp2 (type: timestamp), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), cboolean1 (type: boolean), (cint + UDFToInteger(csmallint)) (type: int), (cbigint - UDFToLong(ctinyint)) (type: bigint), (- cbigint) (type: bigint), (- cfloat) (type: float), ((cbigint - UDFToLong(ctinyint)) + cbigint) (type: bigint), (cdouble / cdouble) (type: double), (- cdouble) (type: double), (UDFToLong((cint + UDFToInteger(csmallint))) * (- cbigint)) (type: bigint), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (-1.389 / CAST( ctinyint AS decimal(3,0))) (type: decimal(8,7)), (UDFToDouble(cbigint) % cdouble) (type: double), (- csmallint) (type: smallint), (UDFToInteger(csmallint) + (cint + UDFToInteger(csmallint))) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), cstring1 (type: string), cboolean2 (type: boolean), ctimestamp2 (type: timestamp), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), cboolean1 (type: boolean), (cint + UDFToInteger(csmallint)) (type: int), (cbigint - UDFToLong(ctinyint)) (type: bigint), (- cbigint) (type: bigint), (- cfloat) (type: float), ((cbigint - UDFToLong(ctinyint)) + cbigint) (type: bigint), (cdouble / cdouble) (type: double), (- cdouble) (type: double), (UDFToLong((cint + UDFToInteger(csmallint))) * (- cbigint)) (type: bigint), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (-1.389 / CAST( ctinyint AS decimal(3,0))) (type: decimal(8,7)), (UDFToDouble(cbigint) % cdouble) (type: double), (- csmallint) (type: smallint), (UDFToInteger(csmallint) + (cint + UDFToInteger(csmallint))) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [2, 6, 11, 9, 5, 4, 3, 1, 10, 19, 20, 21, 22, 24, 25, 26, 29, 32, 34, 36, 37, 39] + selectExpressions: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 19:int, LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 20:bigint, LongColUnaryMinus(col 3:bigint) -> 21:bigint, DoubleColUnaryMinus(col 4:float) -> 22:float, LongColAddLongColumn(col 23:bigint, col 3:bigint)(children: LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 23:bigint) -> 24:bigint, DoubleColDivideDoubleColumn(col 5:double, col 5:double) -> 25:double, DoubleColUnaryMinus(col 5:double) -> 26:double, LongColMultiplyLongColumn(col 27:bigint, col 28:bigint)(children: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 27:int, LongColUnaryMinus(col 3:bigint) -> 28:bigint) -> 29:bigint, DoubleColAddDoubleColumn(col 30:double, col 31:double)(children: DoubleColUnaryMinus(col 5:double) -> 30:double, CastLongToDouble(col 3:bigint) -> 31:double) -> 32:double, DecimalScalarDivideDecimalColumn(val -1.389, col 33:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 33:decimal(3,0)) -> 34:decimal(8,7), DoubleColModuloDoubleColumn(col 35:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 35:double) -> 36:double, LongColUnaryMinus(col 1:smallint) -> 37:smallint, LongColAddLongColumn(col 1:int, col 38:int)(children: col 1:smallint, LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 38:int) -> 39:int + Statistics: Num rows: 10922 Data size: 3483354 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col8 (type: boolean), _col1 (type: string), _col3 (type: timestamp), _col5 (type: float), _col6 (type: bigint), _col4 (type: double), _col0 (type: int), _col7 (type: smallint), _col9 (type: int), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: float), _col13 (type: bigint), _col14 (type: double), _col15 (type: double), _col16 (type: bigint), _col17 (type: double), _col18 (type: decimal(8,7)), _col19 (type: double), _col20 (type: smallint), _col21 (type: int) + null sort order: zzzzzzzzzzzzzzzzzzzzz + sort order: +++++++++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [2, 6, 11, 9, 5, 4, 3, 1, 10, 23, 27, 28, 14, 40, 16, 17, 42, 35, 43, 31, 38, 45] - selectExpressions: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 23:int, LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 27:bigint, LongColUnaryMinus(col 3:bigint) -> 28:bigint, DoubleColUnaryMinus(col 4:float) -> 14:float, LongColAddLongColumn(col 38:bigint, col 3:bigint)(children: LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 38:bigint) -> 40:bigint, DoubleColDivideDoubleColumn(col 5:double, col 5:double) -> 16:double, DoubleColUnaryMinus(col 5:double) -> 17:double, LongColMultiplyLongColumn(col 38:bigint, col 41:bigint)(children: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 38:int, LongColUnaryMinus(col 3:bigint) -> 41:bigint) -> 42:bigint, DoubleColAddDoubleColumn(col 30:double, col 31:double)(children: DoubleColUnaryMinus(col 5:double) -> 30:double, CastLongToDouble(col 3:bigint) -> 31:double) -> 35:double, DecimalScalarDivideDecimalColumn(val -1.389, col 33:decimal(3,0))(children: CastLongToDecimal(col 0:tinyint) -> 33:decimal(3,0)) -> 43:decimal(8,7), DoubleColModuloDoubleColumn(col 30:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 30:double) -> 31:double, LongColUnaryMinus(col 1:smallint) -> 38:smallint, LongColAddLongColumn(col 1:int, col 44:int)(children: col 1:smallint, LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 44:int) -> 45:int - Statistics: Num rows: 10922 Data size: 3012774 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col8 (type: boolean), _col1 (type: string), _col3 (type: timestamp), _col5 (type: float), _col6 (type: bigint), _col4 (type: double), _col0 (type: int), _col7 (type: smallint), _col9 (type: int), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: float), _col13 (type: bigint), _col14 (type: double), _col15 (type: double), _col16 (type: bigint), _col17 (type: double), _col18 (type: decimal(8,7)), _col19 (type: double), _col20 (type: smallint), _col21 (type: int) - null sort order: zzzzzzzzzzzzzzzzzzzzz - sort order: +++++++++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 10922 Data size: 3012774 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: boolean) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 10922 Data size: 3483354 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: boolean) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1763,19 +1736,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [6, 1, 21, 2, 5, 3, 4, 7, 0, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - Statistics: Num rows: 10922 Data size: 3012774 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10922 Data size: 3483354 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 75 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 75 Data size: 20860 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 75 Data size: 24090 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 75 Data size: 20860 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 75 Data size: 24090 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2026,35 +1999,26 @@ STAGE PLANS: predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColGreaterEqualDoubleColumn(col 5:double, col 4:double)(children: col 4:float), FilterStringGroupColLessEqualStringScalar(col 7:string, val a)), FilterExprAndExpr(children: FilterDecimal64ColLessEqualDecimal64Scalar(col 14:decimal(13,3)/DECIMAL_64, val -1389)(children: CastLongToDecimal64(col 2:int) -> 14:decimal(13,3)/DECIMAL_64), FilterLongColLessLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), FilterLongColLessLongScalar(col 1:smallint, val -6432)), FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 6:string, pattern ss%), FilterDecimalColLessDecimalScalar(col 15:decimal(22,3), val 10.175)(children: CastLongToDecimal(col 3:bigint) -> 15:decimal(22,3)))) predicate: (((cdouble >= UDFToDouble(cfloat)) and (cstring2 <= 'a')) or ((CAST( cint AS decimal(13,3)) <= -1.389) and (csmallint < UDFToShort(ctinyint)) and (csmallint < -6432S)) or ((cstring1 like 'ss%') and (CAST( cbigint AS decimal(22,3)) < 10.175))) (type: boolean) Statistics: Num rows: 3828 Data size: 749058 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: +++++++++++++++ - keys: csmallint (type: smallint), cstring2 (type: string), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), (UDFToDouble(cbigint) / 3569.0D) (type: double), (-257 - UDFToInteger(csmallint)) (type: int), (-6432.0 * cfloat) (type: float), (- cdouble) (type: double), (cdouble * 10.175D) (type: double), (UDFToDouble((-6432.0 * cfloat)) / UDFToDouble(cfloat)) (type: double), (- cfloat) (type: float), (cint % UDFToInteger(csmallint)) (type: int), (- cdouble) (type: double), (cdouble * (- cdouble)) (type: double) - null sort order: zzzzzzzzzzzzzzz - Statistics: Num rows: 3828 Data size: 749058 Basic stats: COMPLETE Column stats: COMPLETE - top n: 45 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 1:smallint, col 7:string, col 5:double, col 4:float, col 3:bigint, DoubleColDivideDoubleScalar(col 16:double, val 3569.0)(children: CastLongToDouble(col 3:bigint) -> 16:double) -> 17:double, LongScalarSubtractLongColumn(val -257, col 1:int)(children: col 1:smallint) -> 18:int, DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 19:float, DoubleColUnaryMinus(col 5:double) -> 20:double, DoubleColMultiplyDoubleScalar(col 5:double, val 10.175) -> 21:double, DoubleColDivideDoubleColumn(col 22:double, col 4:double)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 22:float, col 4:float) -> 23:double, DoubleColUnaryMinus(col 4:float) -> 24:float, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 25:int, DoubleColUnaryMinus(col 5:double) -> 26:double, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 27:double) -> 28:double + Select Operator + expressions: ctimestamp1 (type: timestamp), cstring2 (type: string), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), (UDFToDouble(cbigint) / 3569.0D) (type: double), (-257 - UDFToInteger(csmallint)) (type: int), (-6432.0 * cfloat) (type: float), (- cdouble) (type: double), (cdouble * 10.175D) (type: double), (UDFToDouble((-6432.0 * cfloat)) / UDFToDouble(cfloat)) (type: double), (- cfloat) (type: float), (cint % UDFToInteger(csmallint)) (type: int), (- cdouble) (type: double), (cdouble * (- cdouble)) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: ctimestamp1 (type: timestamp), cstring2 (type: string), cdouble (type: double), cfloat (type: float), cbigint (type: bigint), csmallint (type: smallint), (UDFToDouble(cbigint) / 3569.0D) (type: double), (-257 - UDFToInteger(csmallint)) (type: int), (-6432.0 * cfloat) (type: float), (- cdouble) (type: double), (cdouble * 10.175D) (type: double), (UDFToDouble((-6432.0 * cfloat)) / UDFToDouble(cfloat)) (type: double), (- cfloat) (type: float), (cint % UDFToInteger(csmallint)) (type: int), (- cdouble) (type: double), (cdouble * (- cdouble)) (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [8, 7, 5, 4, 3, 1, 17, 18, 19, 20, 21, 23, 24, 25, 26, 28] + selectExpressions: DoubleColDivideDoubleScalar(col 16:double, val 3569.0)(children: CastLongToDouble(col 3:bigint) -> 16:double) -> 17:double, LongScalarSubtractLongColumn(val -257, col 1:int)(children: col 1:smallint) -> 18:int, DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 19:float, DoubleColUnaryMinus(col 5:double) -> 20:double, DoubleColMultiplyDoubleScalar(col 5:double, val 10.175) -> 21:double, DoubleColDivideDoubleColumn(col 22:double, col 4:double)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 22:float, col 4:float) -> 23:double, DoubleColUnaryMinus(col 4:float) -> 24:float, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 25:int, DoubleColUnaryMinus(col 5:double) -> 26:double, DoubleColMultiplyDoubleColumn(col 5:double, col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 27:double) -> 28:double + Statistics: Num rows: 3828 Data size: 702432 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col5 (type: smallint), _col1 (type: string), _col2 (type: double), _col3 (type: float), _col4 (type: bigint), _col6 (type: double), _col7 (type: int), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: float), _col13 (type: int), _col14 (type: double), _col15 (type: double) + null sort order: zzzzzzzzzzzzzzz + sort order: +++++++++++++++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [8, 7, 5, 4, 3, 1, 22, 29, 16, 27, 30, 32, 33, 34, 35, 37] - selectExpressions: DoubleColDivideDoubleScalar(col 16:double, val 3569.0)(children: CastLongToDouble(col 3:bigint) -> 16:double) -> 22:double, LongScalarSubtractLongColumn(val -257, col 1:int)(children: col 1:smallint) -> 29:int, DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 16:float, DoubleColUnaryMinus(col 5:double) -> 27:double, DoubleColMultiplyDoubleScalar(col 5:double, val 10.175) -> 30:double, DoubleColDivideDoubleColumn(col 31:double, col 4:double)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 4:float) -> 31:float, col 4:float) -> 32:double, DoubleColUnaryMinus(col 4:float) -> 33:float, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 34:int, DoubleColUnaryMinus(col 5:double) -> 35:double, DoubleColMultiplyDoubleColumn(col 5:double, col 36:double)(children: DoubleColUnaryMinus(col 5:double) -> 36:double) -> 37:double - Statistics: Num rows: 3828 Data size: 547232 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col5 (type: smallint), _col1 (type: string), _col2 (type: double), _col3 (type: float), _col4 (type: bigint), _col6 (type: double), _col7 (type: int), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: float), _col13 (type: int), _col14 (type: double), _col15 (type: double) - null sort order: zzzzzzzzzzzzzzz - sort order: +++++++++++++++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 3828 Data size: 547232 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: timestamp) + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 3828 Data size: 702432 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: timestamp) Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -2082,19 +2046,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [15, 1, 2, 3, 4, 0, 5, 6, 7, 8, 9, 10, 11, 12, 8, 14] - Statistics: Num rows: 3828 Data size: 547232 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 3828 Data size: 702432 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 45 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 45 Data size: 6560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 45 Data size: 8480 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 45 Data size: 6560 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 45 Data size: 8480 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 980d6cc6c78a..68e5a6fdd77b 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -726,19 +726,17 @@ Stage-0 limit:5 Stage-1 Reducer 2 vectorized - File Output Operator [FS_12] - Limit [LIM_11] (rows=5/5 width=178) + File Output Operator [FS_10] + Limit [LIM_9] (rows=5/5 width=178) Number of rows:5 - Select Operator [SEL_10] (rows=500/8 width=178) + Select Operator [SEL_8] (rows=500/5 width=178) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_9] - Select Operator [SEL_8] (rows=500/36 width=178) + SHUFFLE [RS_7] + Select Operator [SEL_6] (rows=500/500 width=178) Output:["_col0","_col1"] - Top N Key Operator [TNK_7] (rows=500/36 width=178) - keys:key,top n:5 - TableScan [TS_0] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: create table orc_merge5_n1 (userid bigint, string1 string, subtype double, decimal1 decimal(38,0), ts timestamp) stored as orc PREHOOK: type: CREATETABLE diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index cc2adc2fc676..e7eb14b2cc89 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -574,19 +574,17 @@ Stage-0 limit:5 Stage-1 Reducer 2 vectorized - File Output Operator [FS_12] - Limit [LIM_11] (rows=5 width=178) + File Output Operator [FS_10] + Limit [LIM_9] (rows=5 width=178) Number of rows:5 - Select Operator [SEL_10] (rows=500 width=178) + Select Operator [SEL_8] (rows=500 width=178) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_9] - Select Operator [SEL_8] (rows=500 width=178) + SHUFFLE [RS_7] + Select Operator [SEL_6] (rows=500 width=178) Output:["_col0","_col1"] - Top N Key Operator [TNK_7] (rows=500 width=178) - keys:key,top n:5 - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: create table orc_merge5_n0 (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) stored as orc PREHOOK: type: CREATETABLE diff --git a/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out b/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out index fc3a864fd5bb..45f341f18627 100644 --- a/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_non_string_partition.q.out @@ -69,34 +69,25 @@ STAGE PLANS: predicateExpression: FilterLongColGreaterLongScalar(col 0:int, val 0) predicate: (cint > 0) (type: boolean) Statistics: Num rows: 1537 Data size: 12296 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: + - keys: cint (type: int) - null sort order: z - Statistics: Num rows: 1537 Data size: 12296 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int + Select Operator + expressions: cint (type: int), ctinyint (type: tinyint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), ctinyint (type: tinyint) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 4] + Statistics: Num rows: 1537 Data size: 12296 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 4] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1537 Data size: 12296 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int) - null sort order: z - sort order: + - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1537 Data size: 12296 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: tinyint) + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: tinyint) Execution mode: vectorized Map Vectorization: enabled: true @@ -208,33 +199,24 @@ STAGE PLANS: predicateExpression: FilterLongColGreaterLongScalar(col 0:int, val 0) predicate: (cint > 0) (type: boolean) Statistics: Num rows: 1537 Data size: 156774 Basic stats: COMPLETE Column stats: COMPLETE - Top N Key Operator - sort order: ++ - keys: cint (type: int), cstring1 (type: string) - null sort order: zz - Statistics: Num rows: 1537 Data size: 156774 Basic stats: COMPLETE Column stats: COMPLETE - top n: 10 - Top N Key Vectorization: - className: VectorTopNKeyOperator - keyExpressions: col 0:int, col 1:string + Select Operator + expressions: cint (type: int), cstring1 (type: string) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator native: true - Select Operator - expressions: cint (type: int), cstring1 (type: string) - outputColumnNames: _col0, _col1 - Select Vectorization: - className: VectorSelectOperator + projectedOutputColumnNums: [0, 1] + Statistics: Num rows: 1537 Data size: 156774 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + null sort order: zz + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator native: true - projectedOutputColumnNums: [0, 1] + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1537 Data size: 156774 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - null sort order: zz - sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1537 Data size: 156774 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 Execution mode: vectorized Map Vectorization: enabled: true