Skip to content

Commit b7c1df6

Browse files
author
ragupta
committed
HIVE-29376:Using partition spec in DESC FORMATTED sql is unsupported for Iceberg table
1 parent edc5f95 commit b7c1df6

File tree

46 files changed

+627
-28
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+627
-28
lines changed

‎iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -952,8 +952,8 @@ public void testCreatePartitionedTableWithColumnComments() {
952952

953953
List<Object[]> rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table");
954954
List<Types.NestedField> columns = icebergTable.schema().columns();
955-
// The partition transform information is 3 extra lines, and 2 more line for the columns
956-
Assert.assertEquals(columns.size() + 5, rows.size());
955+
// The partition transform information and partition information is 6 extra lines, and 4 more line for the columns
956+
Assert.assertEquals(columns.size() + 10, rows.size());
957957
for (int i = 0; i < columns.size(); i++) {
958958
Types.NestedField field = columns.get(i);
959959
Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
@@ -1316,8 +1316,8 @@ public void testAlterTableRenamePartitionColumn() throws Exception {
13161316
shell.executeStatement("ALTER TABLE default.customers SET PARTITION SPEC (region, city)");
13171317

13181318
List<Object[]> result = shell.executeStatement("DESCRIBE default.customers");
1319-
Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(8));
1320-
Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(9));
1319+
Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(14));
1320+
Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(15));
13211321
}
13221322

13231323
@Test
@@ -1483,25 +1483,6 @@ public void testMetaHookWithUndefinedAlterOperationType() throws Exception {
14831483
metaHook.commitAlterTable(hmsTable, environmentContext);
14841484
}
14851485

1486-
@Test
1487-
public void testCommandsWithPartitionClauseThrow() {
1488-
TableIdentifier target = TableIdentifier.of("default", "target");
1489-
PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA)
1490-
.identity("last_name").build();
1491-
testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
1492-
spec, FileFormat.PARQUET, ImmutableList.of());
1493-
1494-
String[] commands = {
1495-
"DESCRIBE target PARTITION (last_name='Johnson')"
1496-
};
1497-
1498-
for (String command : commands) {
1499-
Assertions.assertThatThrownBy(() -> shell.executeStatement(command))
1500-
.isInstanceOf(IllegalArgumentException.class)
1501-
.hasMessageContaining("Using partition spec in query is unsupported");
1502-
}
1503-
}
1504-
15051486
@Test
15061487
public void testAuthzURIMasked() throws TException, URISyntaxException, InterruptedException {
15071488
testAuthzURI(true);

‎iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
drop table if exists ice_t;
2+
create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg;
3+
insert into table ice_t values( 5, "hello5" ,6, "hello6");
4+
desc formatted ice_t PARTITION(c=6);

‎iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
drop table if exists ice_t;
2+
create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg;
3+
insert into table ice_t values( 5, "hello5" ,6, "hello6");
4+
desc formatted ice_t PARTITION(c=6, d="hello");

‎iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
drop table if exists ice_t;
2+
create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg;
3+
4+
insert into table ice_t values( 1, "hello1" ,2, "hello2");
5+
insert into table ice_t values( 3, "hello3" ,4, "hello4");
6+
insert into table ice_t values( 5, "hello5" ,6, "hello6");
7+
8+
desc extended ice_t PARTITION(c=6,d="hello6");
9+
desc formatted ice_t PARTITION(c=6,d="hello6");

‎iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
PREHOOK: query: drop table if exists ice_t
2+
PREHOOK: type: DROPTABLE
3+
PREHOOK: Output: database:default
4+
POSTHOOK: query: drop table if exists ice_t
5+
POSTHOOK: type: DROPTABLE
6+
POSTHOOK: Output: database:default
7+
PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
8+
PREHOOK: type: CREATETABLE
9+
PREHOOK: Output: database:default
10+
PREHOOK: Output: default@ice_t
11+
POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
12+
POSTHOOK: type: CREATETABLE
13+
POSTHOOK: Output: database:default
14+
POSTHOOK: Output: default@ice_t
15+
PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
16+
PREHOOK: type: QUERY
17+
PREHOOK: Input: _dummy_database@_dummy_table
18+
PREHOOK: Output: default@ice_t
19+
POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
20+
POSTHOOK: type: QUERY
21+
POSTHOOK: Input: _dummy_database@_dummy_table
22+
POSTHOOK: Output: default@ice_t
23+
FAILED: SemanticException [Error 10006]: Partition not found {c=6}

‎iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
PREHOOK: query: drop table if exists ice_t
2+
PREHOOK: type: DROPTABLE
3+
PREHOOK: Output: database:default
4+
POSTHOOK: query: drop table if exists ice_t
5+
POSTHOOK: type: DROPTABLE
6+
POSTHOOK: Output: database:default
7+
PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
8+
PREHOOK: type: CREATETABLE
9+
PREHOOK: Output: database:default
10+
PREHOOK: Output: default@ice_t
11+
POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
12+
POSTHOOK: type: CREATETABLE
13+
POSTHOOK: Output: database:default
14+
POSTHOOK: Output: default@ice_t
15+
PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
16+
PREHOOK: type: QUERY
17+
PREHOOK: Input: _dummy_database@_dummy_table
18+
PREHOOK: Output: default@ice_t
19+
POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
20+
POSTHOOK: type: QUERY
21+
POSTHOOK: Input: _dummy_database@_dummy_table
22+
POSTHOOK: Output: default@ice_t
23+
FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello}

‎iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,11 @@ a int
180180
b string
181181
c string
182182

183+
# Partition Information
184+
# col_name data_type comment
185+
b string Transform: identity
186+
c string Transform: identity
187+
183188
# Partition Transform Information
184189
# col_name transform_type
185190
b IDENTITY
@@ -451,6 +456,11 @@ a int
451456
b string
452457
c string
453458

459+
# Partition Information
460+
# col_name data_type comment
461+
b string Transform: identity
462+
c string Transform: identity
463+
454464
# Partition Transform Information
455465
# col_name transform_type
456466
b IDENTITY
@@ -722,6 +732,11 @@ a int
722732
b string
723733
c string
724734

735+
# Partition Information
736+
# col_name data_type comment
737+
b string Transform: identity
738+
c string Transform: identity
739+
725740
# Partition Transform Information
726741
# col_name transform_type
727742
b IDENTITY
@@ -1055,6 +1070,12 @@ b double
10551070
c int
10561071
d string
10571072

1073+
# Partition Information
1074+
# col_name data_type comment
1075+
b double Transform: identity
1076+
c int Transform: identity
1077+
d string Transform: identity
1078+
10581079
# Partition Transform Information
10591080
# col_name transform_type
10601081
b IDENTITY
@@ -1496,6 +1517,12 @@ b double
14961517
c int
14971518
d string
14981519

1520+
# Partition Information
1521+
# col_name data_type comment
1522+
b double Transform: identity
1523+
c int Transform: identity
1524+
d string Transform: identity
1525+
14991526
# Partition Transform Information
15001527
# col_name transform_type
15011528
b IDENTITY
@@ -1937,6 +1964,12 @@ b double
19371964
c int
19381965
d string
19391966

1967+
# Partition Information
1968+
# col_name data_type comment
1969+
b double Transform: identity
1970+
c int Transform: identity
1971+
d string Transform: identity
1972+
19401973
# Partition Transform Information
19411974
# col_name transform_type
19421975
b IDENTITY

‎iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@ POSTHOOK: Input: default@tbl_orc
139139
a int
140140
b string
141141

142+
# Partition Information
143+
# col_name data_type comment
144+
b string Transform: identity
145+
142146
# Partition Transform Information
143147
# col_name transform_type
144148
b IDENTITY
@@ -413,6 +417,10 @@ POSTHOOK: Input: default@tbl_parquet
413417
a int
414418
b string
415419

420+
# Partition Information
421+
# col_name data_type comment
422+
b string Transform: identity
423+
416424
# Partition Transform Information
417425
# col_name transform_type
418426
b IDENTITY
@@ -764,6 +772,10 @@ POSTHOOK: Input: default@tbl_parquet_int
764772
a int
765773
b int
766774

775+
# Partition Information
776+
# col_name data_type comment
777+
b int Transform: identity
778+
767779
# Partition Transform Information
768780
# col_name transform_type
769781
b IDENTITY
@@ -1115,6 +1127,10 @@ POSTHOOK: Input: default@tbl_parquet_double
11151127
a int
11161128
b double
11171129

1130+
# Partition Information
1131+
# col_name data_type comment
1132+
b double Transform: identity
1133+
11181134
# Partition Transform Information
11191135
# col_name transform_type
11201136
b IDENTITY
@@ -1412,6 +1428,10 @@ POSTHOOK: Input: default@tbl_avro
14121428
a int
14131429
b string
14141430

1431+
# Partition Information
1432+
# col_name data_type comment
1433+
b string Transform: identity
1434+
14151435
# Partition Transform Information
14161436
# col_name transform_type
14171437
b IDENTITY

‎iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,11 @@ a int
277277
b string
278278
c int
279279

280+
# Partition Information
281+
# col_name data_type comment
282+
a int Transform: bucket[16]
283+
b string Transform: truncate[3]
284+
280285
# Partition Transform Information
281286
# col_name transform_type
282287
a BUCKET[16]

‎iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
PREHOOK: query: drop table if exists ice_t
2+
PREHOOK: type: DROPTABLE
3+
PREHOOK: Output: database:default
4+
POSTHOOK: query: drop table if exists ice_t
5+
POSTHOOK: type: DROPTABLE
6+
POSTHOOK: Output: database:default
7+
PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
8+
PREHOOK: type: CREATETABLE
9+
PREHOOK: Output: database:default
10+
PREHOOK: Output: default@ice_t
11+
POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg
12+
POSTHOOK: type: CREATETABLE
13+
POSTHOOK: Output: database:default
14+
POSTHOOK: Output: default@ice_t
15+
PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2")
16+
PREHOOK: type: QUERY
17+
PREHOOK: Input: _dummy_database@_dummy_table
18+
PREHOOK: Output: default@ice_t
19+
POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2")
20+
POSTHOOK: type: QUERY
21+
POSTHOOK: Input: _dummy_database@_dummy_table
22+
POSTHOOK: Output: default@ice_t
23+
PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4")
24+
PREHOOK: type: QUERY
25+
PREHOOK: Input: _dummy_database@_dummy_table
26+
PREHOOK: Output: default@ice_t
27+
POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4")
28+
POSTHOOK: type: QUERY
29+
POSTHOOK: Input: _dummy_database@_dummy_table
30+
POSTHOOK: Output: default@ice_t
31+
PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
32+
PREHOOK: type: QUERY
33+
PREHOOK: Input: _dummy_database@_dummy_table
34+
PREHOOK: Output: default@ice_t
35+
POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6")
36+
POSTHOOK: type: QUERY
37+
POSTHOOK: Input: _dummy_database@_dummy_table
38+
POSTHOOK: Output: default@ice_t
39+
PREHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6")
40+
PREHOOK: type: DESCTABLE
41+
PREHOOK: Input: default@ice_t
42+
POSTHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6")
43+
POSTHOOK: type: DESCTABLE
44+
POSTHOOK: Input: default@ice_t
45+
a int
46+
b string
47+
c int
48+
d string
49+
50+
# Partition Information
51+
# col_name data_type comment
52+
c int Transform: identity
53+
d string Transform: identity
54+
55+
# Partition Transform Information
56+
# col_name transform_type
57+
c IDENTITY
58+
d IDENTITY
59+
60+
#### A masked pattern was here ####
61+
PREHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6")
62+
PREHOOK: type: DESCTABLE
63+
PREHOOK: Input: default@ice_t
64+
POSTHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6")
65+
POSTHOOK: type: DESCTABLE
66+
POSTHOOK: Input: default@ice_t
67+
# col_name data_type comment
68+
a int
69+
b string
70+
c int
71+
d string
72+
73+
# Partition Information
74+
# col_name data_type comment
75+
c int Transform: identity
76+
d string Transform: identity
77+
78+
# Partition Transform Information
79+
# col_name transform_type
80+
c IDENTITY
81+
d IDENTITY
82+
83+
# Detailed Partition Information
84+
Partition Value: [6, hello6]
85+
Database: default
86+
Table: ice_t
87+
#### A masked pattern was here ####
88+
89+
# Storage Information
90+
SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
91+
InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
92+
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
93+
Compressed: No
94+
Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)]

0 commit comments

Comments
 (0)