Skip to content

Commit cc82b9f

Browse files
ueshincloud-fan
ueshin
authored and
cloud-fan
committed
[SPARK-25884][SQL] Add TBLPROPERTIES and COMMENT, and use LOCATION when SHOW CREATE TABLE.
## What changes were proposed in this pull request? When `SHOW CREATE TABLE` for Datasource tables, we are missing `TBLPROPERTIES` and `COMMENT`, and we should use `LOCATION` instead of path in `OPTION`. ## How was this patch tested? Splitted `ShowCreateTableSuite` to confirm to work with both `InMemoryCatalog` and `HiveExternalCatalog`, and added some tests. Closes apache#22892 from ueshin/issues/SPARK-25884/show_create_table. Authored-by: Takuya UESHIN <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent c9667af commit cc82b9f

File tree

7 files changed

+538
-185
lines changed

7 files changed

+538
-185
lines changed

‎sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -957,9 +957,11 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
957957
builder ++= metadata.viewText.mkString(" AS\n", "", "\n")
958958
} else {
959959
showHiveTableHeader(metadata, builder)
960+
showTableComment(metadata, builder)
960961
showHiveTableNonDataColumns(metadata, builder)
961962
showHiveTableStorageInfo(metadata, builder)
962-
showHiveTableProperties(metadata, builder)
963+
showTableLocation(metadata, builder)
964+
showTableProperties(metadata, builder)
963965
}
964966

965967
builder.toString()
@@ -973,14 +975,8 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
973975
if (columns.nonEmpty) {
974976
builder ++= columns.mkString("(", ", ", ")\n")
975977
}
976-
977-
metadata
978-
.comment
979-
.map("COMMENT '" + escapeSingleQuotedString(_) + "'\n")
980-
.foreach(builder.append)
981978
}
982979

983-
984980
private def showHiveTableNonDataColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
985981
if (metadata.partitionColumnNames.nonEmpty) {
986982
val partCols = metadata.partitionSchema.map(_.toDDL)
@@ -1023,15 +1019,24 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
10231019
builder ++= s" OUTPUTFORMAT '${escapeSingleQuotedString(format)}'\n"
10241020
}
10251021
}
1022+
}
10261023

1024+
private def showTableLocation(metadata: CatalogTable, builder: StringBuilder): Unit = {
10271025
if (metadata.tableType == EXTERNAL) {
1028-
storage.locationUri.foreach { uri =>
1029-
builder ++= s"LOCATION '$uri'\n"
1026+
metadata.storage.locationUri.foreach { location =>
1027+
builder ++= s"LOCATION '${escapeSingleQuotedString(CatalogUtils.URIToString(location))}'\n"
10301028
}
10311029
}
10321030
}
10331031

1034-
private def showHiveTableProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
1032+
private def showTableComment(metadata: CatalogTable, builder: StringBuilder): Unit = {
1033+
metadata
1034+
.comment
1035+
.map("COMMENT '" + escapeSingleQuotedString(_) + "'\n")
1036+
.foreach(builder.append)
1037+
}
1038+
1039+
private def showTableProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
10351040
if (metadata.properties.nonEmpty) {
10361041
val props = metadata.properties.map { case (key, value) =>
10371042
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
@@ -1048,6 +1053,9 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
10481053
showDataSourceTableDataColumns(metadata, builder)
10491054
showDataSourceTableOptions(metadata, builder)
10501055
showDataSourceTableNonDataColumns(metadata, builder)
1056+
showTableComment(metadata, builder)
1057+
showTableLocation(metadata, builder)
1058+
showTableProperties(metadata, builder)
10511059

10521060
builder.toString()
10531061
}
@@ -1063,14 +1071,6 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
10631071

10641072
val dataSourceOptions = metadata.storage.properties.map {
10651073
case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'"
1066-
} ++ metadata.storage.locationUri.flatMap { location =>
1067-
if (metadata.tableType == MANAGED) {
1068-
// If it's a managed table, omit PATH option. Spark SQL always creates external table
1069-
// when the table creation DDL contains the PATH option.
1070-
None
1071-
} else {
1072-
Some(s"path '${escapeSingleQuotedString(CatalogUtils.URIToString(location))}'")
1073-
}
10741074
}
10751075

10761076
if (dataSourceOptions.nonEmpty) {

‎sql/hive/src/test/resources/sample.json renamed to ‎sql/core/src/test/resources/sample.json

File renamed without changes.

‎sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
-- simple
2+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet;
3+
4+
SHOW CREATE TABLE tbl;
5+
DROP TABLE tbl;
6+
7+
8+
-- options
9+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
10+
OPTIONS ('a' 1);
11+
12+
SHOW CREATE TABLE tbl;
13+
DROP TABLE tbl;
14+
15+
16+
-- path option
17+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
18+
OPTIONS ('path' '/path/to/table');
19+
20+
SHOW CREATE TABLE tbl;
21+
DROP TABLE tbl;
22+
23+
24+
-- location
25+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
26+
LOCATION '/path/to/table';
27+
28+
SHOW CREATE TABLE tbl;
29+
DROP TABLE tbl;
30+
31+
32+
-- partition by
33+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
34+
PARTITIONED BY (a);
35+
36+
SHOW CREATE TABLE tbl;
37+
DROP TABLE tbl;
38+
39+
40+
-- clustered by
41+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
42+
CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS;
43+
44+
SHOW CREATE TABLE tbl;
45+
DROP TABLE tbl;
46+
47+
48+
-- comment
49+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
50+
COMMENT 'This is a comment';
51+
52+
SHOW CREATE TABLE tbl;
53+
DROP TABLE tbl;
54+
55+
56+
-- tblproperties
57+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
58+
TBLPROPERTIES ('a' = '1');
59+
60+
SHOW CREATE TABLE tbl;
61+
DROP TABLE tbl;

‎sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
-- Automatically generated by SQLQueryTestSuite
2+
-- Number of queries: 24
3+
4+
5+
-- !query 0
6+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
7+
-- !query 0 schema
8+
struct<>
9+
-- !query 0 output
10+
11+
12+
13+
-- !query 1
14+
SHOW CREATE TABLE tbl
15+
-- !query 1 schema
16+
struct<createtab_stmt:string>
17+
-- !query 1 output
18+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
19+
USING parquet
20+
21+
22+
-- !query 2
23+
DROP TABLE tbl
24+
-- !query 2 schema
25+
struct<>
26+
-- !query 2 output
27+
28+
29+
30+
-- !query 3
31+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
32+
OPTIONS ('a' 1)
33+
-- !query 3 schema
34+
struct<>
35+
-- !query 3 output
36+
37+
38+
39+
-- !query 4
40+
SHOW CREATE TABLE tbl
41+
-- !query 4 schema
42+
struct<createtab_stmt:string>
43+
-- !query 4 output
44+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
45+
USING parquet
46+
OPTIONS (
47+
`a` '1'
48+
)
49+
50+
51+
-- !query 5
52+
DROP TABLE tbl
53+
-- !query 5 schema
54+
struct<>
55+
-- !query 5 output
56+
57+
58+
59+
-- !query 6
60+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
61+
OPTIONS ('path' '/path/to/table')
62+
-- !query 6 schema
63+
struct<>
64+
-- !query 6 output
65+
66+
67+
68+
-- !query 7
69+
SHOW CREATE TABLE tbl
70+
-- !query 7 schema
71+
struct<createtab_stmt:string>
72+
-- !query 7 output
73+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
74+
USING parquet
75+
LOCATION 'file:/path/to/table'
76+
77+
78+
-- !query 8
79+
DROP TABLE tbl
80+
-- !query 8 schema
81+
struct<>
82+
-- !query 8 output
83+
84+
85+
86+
-- !query 9
87+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
88+
LOCATION '/path/to/table'
89+
-- !query 9 schema
90+
struct<>
91+
-- !query 9 output
92+
93+
94+
95+
-- !query 10
96+
SHOW CREATE TABLE tbl
97+
-- !query 10 schema
98+
struct<createtab_stmt:string>
99+
-- !query 10 output
100+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
101+
USING parquet
102+
LOCATION 'file:/path/to/table'
103+
104+
105+
-- !query 11
106+
DROP TABLE tbl
107+
-- !query 11 schema
108+
struct<>
109+
-- !query 11 output
110+
111+
112+
113+
-- !query 12
114+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
115+
PARTITIONED BY (a)
116+
-- !query 12 schema
117+
struct<>
118+
-- !query 12 output
119+
120+
121+
122+
-- !query 13
123+
SHOW CREATE TABLE tbl
124+
-- !query 13 schema
125+
struct<createtab_stmt:string>
126+
-- !query 13 output
127+
CREATE TABLE `tbl` (`b` STRING, `c` INT, `a` INT)
128+
USING parquet
129+
PARTITIONED BY (a)
130+
131+
132+
-- !query 14
133+
DROP TABLE tbl
134+
-- !query 14 schema
135+
struct<>
136+
-- !query 14 output
137+
138+
139+
140+
-- !query 15
141+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
142+
CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
143+
-- !query 15 schema
144+
struct<>
145+
-- !query 15 output
146+
147+
148+
149+
-- !query 16
150+
SHOW CREATE TABLE tbl
151+
-- !query 16 schema
152+
struct<createtab_stmt:string>
153+
-- !query 16 output
154+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
155+
USING parquet
156+
CLUSTERED BY (a)
157+
SORTED BY (b)
158+
INTO 2 BUCKETS
159+
160+
161+
-- !query 17
162+
DROP TABLE tbl
163+
-- !query 17 schema
164+
struct<>
165+
-- !query 17 output
166+
167+
168+
169+
-- !query 18
170+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
171+
COMMENT 'This is a comment'
172+
-- !query 18 schema
173+
struct<>
174+
-- !query 18 output
175+
176+
177+
178+
-- !query 19
179+
SHOW CREATE TABLE tbl
180+
-- !query 19 schema
181+
struct<createtab_stmt:string>
182+
-- !query 19 output
183+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
184+
USING parquet
185+
COMMENT 'This is a comment'
186+
187+
188+
-- !query 20
189+
DROP TABLE tbl
190+
-- !query 20 schema
191+
struct<>
192+
-- !query 20 output
193+
194+
195+
196+
-- !query 21
197+
CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
198+
TBLPROPERTIES ('a' = '1')
199+
-- !query 21 schema
200+
struct<>
201+
-- !query 21 output
202+
203+
204+
205+
-- !query 22
206+
SHOW CREATE TABLE tbl
207+
-- !query 22 schema
208+
struct<createtab_stmt:string>
209+
-- !query 22 output
210+
CREATE TABLE `tbl` (`a` INT, `b` STRING, `c` INT)
211+
USING parquet
212+
TBLPROPERTIES (
213+
'a' = '1'
214+
)
215+
216+
217+
-- !query 23
218+
DROP TABLE tbl
219+
-- !query 23 schema
220+
struct<>
221+
-- !query 23 output
222+

0 commit comments

Comments
 (0)