Skip to content

Commit 7fc8881

Browse files
committed
[SPARK-25296][SQL][TEST] Create ExplainSuite
## What changes were proposed in this pull request? Move the output verification of Explain test cases to a new suite ExplainSuite. ## How was this patch tested? N/A Closes apache#22300 from gatorsmile/test3200. Authored-by: Xiao Li <[email protected]> Signed-off-by: Xiao Li <[email protected]>
1 parent 339859c commit 7fc8881

File tree

4 files changed

+58
-30
lines changed

4 files changed

+58
-30
lines changed

sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2553,13 +2553,4 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
25532553
}
25542554
}
25552555

2556-
test("SPARK-23034 show rdd names in RDD scan nodes") {
2557-
val rddWithName = spark.sparkContext.parallelize(Row(1, "abc") :: Nil).setName("testRdd")
2558-
val df2 = spark.createDataFrame(rddWithName, StructType.fromDDL("c0 int, c1 string"))
2559-
val output2 = new java.io.ByteArrayOutputStream()
2560-
Console.withOut(output2) {
2561-
df2.explain(extended = false)
2562-
}
2563-
assert(output2.toString.contains("Scan ExistingRDD testRdd"))
2564-
}
25652556
}

sql/core/src/test/scala/org/apache/spark/sql/DatasetCacheSuite.scala

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -206,15 +206,4 @@ class DatasetCacheSuite extends QueryTest with SharedSQLContext with TimeLimits
206206
// first time use, load cache
207207
checkDataset(df5, Row(10))
208208
}
209-
210-
test("SPARK-24850 InMemoryRelation string representation does not include cached plan") {
211-
val df = Seq(1).toDF("a").cache()
212-
val outputStream = new java.io.ByteArrayOutputStream()
213-
Console.withOut(outputStream) {
214-
df.explain(false)
215-
}
216-
assert(outputStream.toString.replaceAll("#\\d+", "#x").contains(
217-
"InMemoryRelation [a#x], StorageLevel(disk, memory, deserialized, 1 replicas)"
218-
))
219-
}
220209
}

sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1498,16 +1498,6 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
14981498
df.where($"city".contains(new java.lang.Character('A'))),
14991499
Seq(Row("Amsterdam")))
15001500
}
1501-
1502-
test("SPARK-23034 show rdd names in RDD scan nodes") {
1503-
val rddWithName = spark.sparkContext.parallelize(SingleData(1) :: Nil).setName("testRdd")
1504-
val df = spark.createDataFrame(rddWithName)
1505-
val output = new java.io.ByteArrayOutputStream()
1506-
Console.withOut(output) {
1507-
df.explain(extended = false)
1508-
}
1509-
assert(output.toString.contains("Scan testRdd"))
1510-
}
15111501
}
15121502

15131503
case class TestDataUnion(x: Int, y: Int, z: Int)
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql
19+
20+
import org.apache.spark.sql.test.SharedSQLContext
21+
import org.apache.spark.sql.types.StructType
22+
23+
class ExplainSuite extends QueryTest with SharedSQLContext {
24+
import testImplicits._
25+
26+
/**
27+
* Runs the plan and makes sure the plans contains all of the keywords.
28+
*/
29+
private def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
30+
val output = new java.io.ByteArrayOutputStream()
31+
Console.withOut(output) {
32+
df.explain(extended = false)
33+
}
34+
for (key <- keywords) {
35+
assert(output.toString.contains(key))
36+
}
37+
}
38+
39+
test("SPARK-23034 show rdd names in RDD scan nodes (Dataset)") {
40+
val rddWithName = spark.sparkContext.parallelize(Row(1, "abc") :: Nil).setName("testRdd")
41+
val df = spark.createDataFrame(rddWithName, StructType.fromDDL("c0 int, c1 string"))
42+
checkKeywordsExistsInExplain(df, keywords = "Scan ExistingRDD testRdd")
43+
}
44+
45+
test("SPARK-23034 show rdd names in RDD scan nodes (DataFrame)") {
46+
val rddWithName = spark.sparkContext.parallelize(ExplainSingleData(1) :: Nil).setName("testRdd")
47+
val df = spark.createDataFrame(rddWithName)
48+
checkKeywordsExistsInExplain(df, keywords = "Scan testRdd")
49+
}
50+
51+
test("SPARK-24850 InMemoryRelation string representation does not include cached plan") {
52+
val df = Seq(1).toDF("a").cache()
53+
checkKeywordsExistsInExplain(df,
54+
keywords = "InMemoryRelation", "StorageLevel(disk, memory, deserialized, 1 replicas)")
55+
}
56+
}
57+
58+
case class ExplainSingleData(id: Int)

0 commit comments

Comments
 (0)