Skip to content

Commit 9e10f69

Browse files
jinxingcloud-fan
jinxing
authored and
cloud-fan
committed
[SPARK-22676][FOLLOW-UP] fix code style for test.
## What changes were proposed in this pull request? This pr address comments in apache#19868 ; Fix the code style for `org.apache.spark.sql.hive.QueryPartitionSuite` by using: `withTempView`, `withTempDir`, `withTable`... Author: jinxing <[email protected]> Closes apache#21091 from jinxing64/SPARK-22676-FOLLOW-UP.
1 parent e134165 commit 9e10f69

File tree

1 file changed

+41
-68
lines changed

1 file changed

+41
-68
lines changed

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala

Lines changed: 41 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -33,80 +33,53 @@ import org.apache.spark.util.Utils
3333
class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
3434
import spark.implicits._
3535

36-
test("SPARK-5068: query data when path doesn't exist") {
37-
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "true")) {
38-
val testData = sparkContext.parallelize(
39-
(1 to 10).map(i => TestData(i, i.toString))).toDF()
40-
testData.createOrReplaceTempView("testData")
41-
42-
val tmpDir = Files.createTempDir()
43-
// create the table for test
44-
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
45-
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
46-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
47-
"SELECT key,value FROM testData")
48-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
49-
"SELECT key,value FROM testData")
50-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
51-
"SELECT key,value FROM testData")
52-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
53-
"SELECT key,value FROM testData")
54-
55-
// test for the exist path
56-
checkAnswer(sql("select key,value from table_with_partition"),
57-
testData.toDF.collect ++ testData.toDF.collect
58-
++ testData.toDF.collect ++ testData.toDF.collect)
59-
60-
// delete the path of one partition
61-
tmpDir.listFiles
62-
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
63-
.foreach { f => Utils.deleteRecursively(f) }
64-
65-
// test for after delete the path
66-
checkAnswer(sql("select key,value from table_with_partition"),
67-
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
36+
private def queryWhenPathNotExist(): Unit = {
37+
withTempView("testData") {
38+
withTable("table_with_partition", "createAndInsertTest") {
39+
withTempDir { tmpDir =>
40+
val testData = sparkContext.parallelize(
41+
(1 to 10).map(i => TestData(i, i.toString))).toDF()
42+
testData.createOrReplaceTempView("testData")
43+
44+
// create the table for test
45+
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
46+
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
47+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
48+
"SELECT key,value FROM testData")
49+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
50+
"SELECT key,value FROM testData")
51+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
52+
"SELECT key,value FROM testData")
53+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
54+
"SELECT key,value FROM testData")
55+
56+
// test for the exist path
57+
checkAnswer(sql("select key,value from table_with_partition"),
58+
testData.union(testData).union(testData).union(testData))
59+
60+
// delete the path of one partition
61+
tmpDir.listFiles
62+
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
63+
.foreach { f => Utils.deleteRecursively(f) }
64+
65+
// test for after delete the path
66+
checkAnswer(sql("select key,value from table_with_partition"),
67+
testData.union(testData).union(testData))
68+
}
69+
}
70+
}
71+
}
6872

69-
sql("DROP TABLE IF EXISTS table_with_partition")
70-
sql("DROP TABLE IF EXISTS createAndInsertTest")
73+
test("SPARK-5068: query data when path doesn't exist") {
74+
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true") {
75+
queryWhenPathNotExist()
7176
}
7277
}
7378

7479
test("Replace spark.sql.hive.verifyPartitionPath by spark.files.ignoreMissingFiles") {
75-
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "false")) {
80+
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "false") {
7681
sparkContext.conf.set(IGNORE_MISSING_FILES.key, "true")
77-
val testData = sparkContext.parallelize(
78-
(1 to 10).map(i => TestData(i, i.toString))).toDF()
79-
testData.createOrReplaceTempView("testData")
80-
81-
val tmpDir = Files.createTempDir()
82-
// create the table for test
83-
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
84-
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
85-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
86-
"SELECT key,value FROM testData")
87-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
88-
"SELECT key,value FROM testData")
89-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
90-
"SELECT key,value FROM testData")
91-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
92-
"SELECT key,value FROM testData")
93-
94-
// test for the exist path
95-
checkAnswer(sql("select key,value from table_with_partition"),
96-
testData.toDF.collect ++ testData.toDF.collect
97-
++ testData.toDF.collect ++ testData.toDF.collect)
98-
99-
// delete the path of one partition
100-
tmpDir.listFiles
101-
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
102-
.foreach { f => Utils.deleteRecursively(f) }
103-
104-
// test for after delete the path
105-
checkAnswer(sql("select key,value from table_with_partition"),
106-
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
107-
108-
sql("DROP TABLE IF EXISTS table_with_partition")
109-
sql("DROP TABLE IF EXISTS createAndInsertTest")
82+
queryWhenPathNotExist()
11083
}
11184
}
11285

0 commit comments

Comments
 (0)