diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 2978341b6da5..b21e7c8f55fe 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2075,11 +2075,31 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) return table.spec().isPartitioned(); } + private boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec) throws SemanticException { + try { + List partNames = getPartitionNames(table, partitionSpec); + for (String partName : partNames) { + if (Warehouse.makeSpecFromName(partName).size() == partitionSpec.size()) { + return true; + } + } + } catch (HiveException e) { + return false; + } catch (MetaException e) { + throw new SemanticException("Unable to construct spec for partition name due to: ", e); + } + return false; + } + @Override public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { validatePartSpec(table, partitionSpec, policy); try { + if (!isPartitionPresent(table, partitionSpec)) { + return null; + } String partName = Warehouse.makePartName(partitionSpec, false); return new DummyPartition(table, partName, partitionSpec); } catch (MetaException e) { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index b6d39bd01a25..fea4996e30bb 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -952,8 +952,8 @@ public void testCreatePartitionedTableWithColumnComments() { List rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table"); List columns = icebergTable.schema().columns(); - // The partition transform information is 3 extra lines, and 2 more line for the columns - Assert.assertEquals(columns.size() + 5, rows.size()); + // The partition transform information and partition information is 6 extra lines, and 4 more line for the columns + Assert.assertEquals(columns.size() + 10, rows.size()); for (int i = 0; i < columns.size(); i++) { Types.NestedField field = columns.get(i); Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), @@ -1316,8 +1316,8 @@ public void testAlterTableRenamePartitionColumn() throws Exception { shell.executeStatement("ALTER TABLE default.customers SET PARTITION SPEC (region, city)"); List result = shell.executeStatement("DESCRIBE default.customers"); - Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(8)); - Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(9)); + Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(14)); + Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(15)); } @Test @@ -1483,25 +1483,6 @@ public void testMetaHookWithUndefinedAlterOperationType() throws Exception { metaHook.commitAlterTable(hmsTable, environmentContext); } - @Test - public void testCommandsWithPartitionClauseThrow() { - TableIdentifier target = TableIdentifier.of("default", "target"); - PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) - .identity("last_name").build(); - testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, - spec, FileFormat.PARQUET, ImmutableList.of()); - - String[] commands = { - "DESCRIBE target PARTITION (last_name='Johnson')" - }; - - for (String command : commands) { - Assertions.assertThatThrownBy(() -> shell.executeStatement(command)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Using partition spec in query is unsupported"); - } - } - @Test public void testAuthzURIMasked() throws TException, URISyntaxException, InterruptedException { testAuthzURI(true); diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q new file mode 100644 index 000000000000..577785fd6956 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q @@ -0,0 +1,4 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; +insert into table ice_t values( 5, "hello5" ,6, "hello6"); +desc formatted ice_t PARTITION(c=6); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q new file mode 100644 index 000000000000..974e61c14c1b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q @@ -0,0 +1,4 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; +insert into table ice_t values( 5, "hello5" ,6, "hello6"); +desc formatted ice_t PARTITION(c=6, d="hello"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q new file mode 100644 index 000000000000..1246cecec7ed --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -0,0 +1,14 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; + +insert into table ice_t values( 1, "hello1" ,2, "hello2" ); +insert into table ice_t values( 3, "hello3" ,4, "hello4" ); +insert into table ice_t values( 5, "hello5" ,6, "hello6" ); + +desc extended ice_t PARTITION( c=6, d="hello6" ); +desc formatted ice_t PARTITION( c=6, d="hello6" ); + +alter table ice_t set partition spec ( c, d, b ); +insert into table ice_t values( 7, "hello7" , 8 , "hello8" ); +desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ); +desc formatted ice_t PARTITION( c=4 , d="hello4" ); diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out new file mode 100644 index 000000000000..0165c68d6245 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out @@ -0,0 +1,23 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6} diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out new file mode 100644 index 000000000000..fb6e7f7acf45 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out @@ -0,0 +1,23 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello} diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out index 118552ac4d5d..910e48e4214e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out @@ -180,6 +180,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -451,6 +456,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -722,6 +732,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1055,6 +1070,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1496,6 +1517,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1937,6 +1964,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out index 2e55bbd42b7c..55bfee6eb031 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out @@ -139,6 +139,10 @@ POSTHOOK: Input: default@tbl_orc a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -413,6 +417,10 @@ POSTHOOK: Input: default@tbl_parquet a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -764,6 +772,10 @@ POSTHOOK: Input: default@tbl_parquet_int a int b int +# Partition Information +# col_name data_type comment +b int Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1115,6 +1127,10 @@ POSTHOOK: Input: default@tbl_parquet_double a int b double +# Partition Information +# col_name data_type comment +b double Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1412,6 +1428,10 @@ POSTHOOK: Input: default@tbl_avro a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index c0a81219df6d..0d1700ff07a9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -277,6 +277,11 @@ a int b string c int +# Partition Information +# col_name data_type comment +a int Transform: bucket[16] +b string Transform: truncate[3] + # Partition Transform Information # col_name transform_type a BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out new file mode 100644 index 000000000000..92f587f9b519 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -0,0 +1,181 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +# Detailed Partition Information +Partition Value: [6, hello6] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: alter table ice_t set partition spec ( c, d, b ) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: alter table ice_t set partition spec ( c, d, b ) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity +b string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY +b IDENTITY + +# Detailed Partition Information +Partition Value: [8, hello8, hello7] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity +b string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY +b IDENTITY + +# Detailed Partition Information +Partition Value: [4, hello4, null] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index 09ef4d1ac96c..fb1cdbcaf12a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -117,6 +117,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -180,6 +190,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -237,6 +257,10 @@ POSTHOOK: Input: default@ice_t_identity_part a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out index c7e7c4e4acf7..74d891eefc1e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out @@ -30,6 +30,11 @@ intcol int pcol string datecol date +# Partition Information +# col_name data_type comment +pcol string Transform: identity +datecol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index 000b8b0ba819..c68cb256cd60 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -481,6 +481,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1675,6 +1679,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -2727,6 +2736,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -3418,6 +3432,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4108,6 +4126,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4798,6 +4820,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index 1cbaf6d004bf..fcd1c17b24ed 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -612,6 +612,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1304,6 +1308,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1996,6 +2004,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2447,6 +2459,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] @@ -2882,6 +2898,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] @@ -3105,6 +3125,10 @@ POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket # col_name data_type comment pcol decimal(38,0) +# Partition Information +# col_name data_type comment +pcol decimal(38,0) Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 0a724298b074..de49a0d3b366 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -164,6 +164,10 @@ POSTHOOK: Input: default@testice1000 a int b string +# Partition Information +# col_name data_type comment +b string Transform: truncate[2] + # Partition Transform Information # col_name transform_type b TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 7aaec8f1924e..063b6389863e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -263,6 +263,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1231,6 +1235,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -1708,6 +1717,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2172,6 +2185,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2636,6 +2653,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -3100,6 +3121,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 7b98a5349057..12745b17f098 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -608,6 +608,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1274,6 +1278,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1944,6 +1952,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2395,6 +2407,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index c6a7243ec961..5036ca420f88 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -92,6 +92,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -170,6 +175,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -256,6 +266,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -340,6 +355,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -410,6 +430,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index b0b02dda8a84..2915bf7d774d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -273,6 +273,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -865,6 +870,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index d0195d0d475e..3af63c58b6c2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -140,6 +140,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -223,6 +227,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index 4b5738e6b7a9..924c6d9953fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -134,6 +134,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +284,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 1f6be0ebf87a..61a931d74a33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -228,6 +228,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] @@ -334,6 +339,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index 96f9e6fe34c1..bc786c1e8dc8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -180,6 +180,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -288,6 +293,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out index ef881a53ba39..5508bdca124c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out @@ -175,6 +175,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -274,6 +278,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -508,6 +516,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -611,6 +623,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out index 9e59529a9e2f..440f6334f114 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out @@ -211,6 +211,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -311,6 +315,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out index ba16b014e3c6..4e120fb8c50d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out @@ -181,6 +181,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -287,6 +293,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -400,6 +412,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out index 23ff2c3bafa8..1d1143f4b635 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out @@ -162,6 +162,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -273,6 +279,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -384,6 +396,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -494,6 +512,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -604,6 +628,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index f3a5e2964b5a..b01185bb6911 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -109,6 +109,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -199,6 +203,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -303,6 +311,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -407,6 +419,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 2a1860b2b3d5..fa5dcd05e101 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -68,6 +68,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] @@ -182,6 +186,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 52ef77680462..62070278505d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -96,6 +96,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -175,6 +179,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +288,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index e31fb1f787e2..5f31e752db3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -881,6 +881,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index 1902755ec3a1..f1a017215639 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -610,6 +610,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index bbd6c589e90a..2feda580b67a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -513,6 +513,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out index 0bb8f5837f73..7ea7605467ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out @@ -50,6 +50,10 @@ POSTHOOK: Input: default@mat1 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -138,6 +142,10 @@ POSTHOOK: Input: default@mat2 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out index d08fb3ec1095..770cc967d0bc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out @@ -50,6 +50,11 @@ POSTHOOK: Input: default@mat1 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] @@ -139,6 +144,11 @@ POSTHOOK: Input: default@mat2 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 1c4d2e05cdb8..a44d9394025b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -85,6 +85,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -172,6 +177,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out index c3c309b656aa..24fff78df880 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out @@ -37,6 +37,11 @@ equality_delete_record_count bigint Count of records in equality d equality_delete_file_count int Count of equality delete files last_updated_at timestamp with local time zone Commit time of snapshot that last updated this partition last_updated_snapshot_id bigint Id of snapshot that last updated this partition + +# Partition Information +# col_name data_type comment +d_part int Transform: identity +e_part int Transform: identity PREHOOK: query: select * from default.ice1.partitions PREHOOK: type: QUERY PREHOOK: Input: default@ice1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out index 0d6f6a02a622..be765c27120a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out @@ -81,6 +81,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -185,6 +189,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index f37b6a27174a..cd0ce562a725 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -762,6 +762,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 95a19640d864..fdf2679f5b2a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -531,6 +531,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index 4f3d0dd65dda..acc7794e12ce 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -435,6 +435,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index db1e6a6abbac..53edd29ce0be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -149,20 +149,18 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta if (node.getChild(1).getType() == HiveParser.TOK_PARTSPEC) { ASTNode partNode = (ASTNode) node.getChild(1); - Table tab = null; + Table table; try { - tab = db.getTable(tableName.getNotEmptyDbTable()); + table = db.getTable(tableName.getNotEmptyDbTable()); } catch (InvalidTableException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName.getNotEmptyDbTable()), e); } catch (HiveException e) { throw new SemanticException(e.getMessage(), e); } - Map partitionSpec = null; + Map partitionSpec; try { - partitionSpec = getPartSpec(partNode); - validateUnsupportedPartitionClause(tab, partitionSpec != null && !partitionSpec.isEmpty()); - partitionSpec = getValidatedPartSpec(tab, partNode, db.getConf(), false); + partitionSpec = getValidatedPartSpec(table, partNode, db.getConf(), false); } catch (SemanticException e) { // get exception in resolving partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key @@ -170,9 +168,9 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } if (partitionSpec != null) { - Partition part = null; + Partition part; try { - part = db.getPartition(tab, partitionSpec, false); + part = db.getPartition(table, partitionSpec); } catch (HiveException e) { // if get exception in finding partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 9f9f9351ef48..6fef8f953ac4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -127,7 +127,7 @@ private Table getTable() throws HiveException { private Partition getPartition(Table table) throws HiveException { Partition part = null; if (desc.getPartitionSpec() != null) { - part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); + part = context.getDb().getPartition(table, desc.getPartitionSpec()); if (part == null) { throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(desc.getPartitionSpec().keySet(), ','), desc.getDbTableName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 6f73c36b6ad1..2847f182a8a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -171,7 +171,12 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - List partitionColumns = table.isPartitioned() ? table.getPartCols() : null; + List partitionColumns = null; + if (table.isPartitioned()) { + partitionColumns = table.hasNonNativePartitionSupport() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols(); + } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 89681572c952..1e488221e44a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -56,6 +56,7 @@ public DummyPartition(Table tbl, String name, Map partSpec) { new org.apache.hadoop.hive.metastore.api.Partition(); tPart.setSd(tbl.getSd().deepCopy()); tPart.setParameters(Maps.newHashMap()); + tPart.setDbName(tbl.getDbName()); this.partSpec = Maps.newLinkedHashMap(partSpec); setTPartition(tPart); @@ -80,8 +81,12 @@ public LinkedHashMap getSpec() { @Override public List getValues() { + Table table = this.getTable(); List values = new ArrayList(); - for (FieldSchema fs : this.getTable().getPartCols()) { + for (FieldSchema fs : + table.hasNonNativePartitionSupport() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols()) { values.add(partSpec.get(fs.getName())); } return values; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 393006bd8f06..0131f1f9629e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.DataConnector; import org.apache.hadoop.hive.metastore.api.Database; @@ -59,6 +60,7 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; @@ -87,6 +89,7 @@ import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; @@ -1345,7 +1348,14 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit if (partHandle == null) { // if partSpec doesn't exists in DB, return a delegate one // and the actual partition is created in MoveTask - partHandle = new Partition(tableHandle, partSpec, null); + try { + partHandle = tableHandle.hasNonNativePartitionSupport() ? + new DummyPartition(tableHandle, Warehouse.makePartName(partSpec, false), partSpec) : + new Partition(tableHandle, partSpec, null); + } catch (MetaException e) { + throw new SemanticException("Unable to construct name for dummy partition due to: ", e); + } + } else { partitions.add(partHandle); } @@ -1739,23 +1749,6 @@ public static void validatePartSpec(Table tbl, Map partSpec, } } - /** - * Throws an UnsupportedOperationException in case the query has a partition clause but the table is never partitioned - * on the HMS-level. Even though table is not partitioned from the HMS's point of view, it might have some other - * notion of partitioning under the hood (e.g. Iceberg tables). In these cases, we might decide to proactively throw a - * more descriptive, unified error message instead of failing on some other semantic analysis validation step, which - * could provide a more counter-intuitive exception message. - * - * @param tbl The table object, should not be null. - * @param partitionClausePresent Whether a partition clause is present in the query (e.g. PARTITION(last_name='Don')) - */ - protected static void validateUnsupportedPartitionClause(Table tbl, boolean partitionClausePresent) { - if (partitionClausePresent && tbl.hasNonNativePartitionSupport()) { - throw new UnsupportedOperationException("Using partition spec in query is unsupported for non-native table" + - " backed by: " + tbl.getStorageHandler().toString()); - } - } - public static void validatePartColumnType(Table tbl, Map partSpec, ASTNode astNode, HiveConf conf) throws SemanticException { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {