Skip to content

Commit 32922c9

Browse files
committed
Re-arranged some test cases
1. Removed meaningless test cases using the remote example jar as the tests will succeed even if there's any issue with downloading and mounting the remote example jar, as the example jar is available locally in the containers. 2. Added a new test case using the PageRank example with a remote data file and test secret mounted.
1 parent cc17bd4 commit 32922c9

File tree

1 file changed

+24
-24
lines changed

1 file changed

+24
-24
lines changed

integration-test/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,12 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
4040
private var kubernetesTestComponents: KubernetesTestComponents = _
4141
private var sparkAppConf: SparkAppConf = _
4242

43-
private val driverImage = System.getProperty("spark.docker.test.driverImage", "spark-driver:latest")
44-
private val executorImage = System.getProperty("spark.docker.test.executorImage", "spark-executor:latest")
45-
private val initContainerImage = System.getProperty("spark.docker.test.initContainerImage", "spark-init:latest")
43+
private val driverImage = System.getProperty("spark.docker.test.driverImage",
44+
"spark-driver:latest")
45+
private val executorImage = System.getProperty("spark.docker.test.executorImage",
46+
"spark-executor:latest")
47+
private val initContainerImage = System.getProperty("spark.docker.test.initContainerImage",
48+
"spark-init:latest")
4649

4750

4851
override def beforeAll(): Unit = {
@@ -91,11 +94,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
9194
runSparkPiAndVerifyCompletion(appArgs = Array("5"))
9295
}
9396

94-
test("Run SparkPi using the remote example jar.") {
95-
sparkAppConf.set("spark.kubernetes.initContainer.image", initContainerImage)
96-
runSparkPiAndVerifyCompletion(appResource = REMOTE_EXAMPLES_JAR_URI)
97-
}
98-
9997
test("Run SparkPi with custom driver pod name, labels, annotations, and environment variables.") {
10098
sparkAppConf
10199
.set("spark.kubernetes.driver.pod.name", "spark-integration-spark-pi")
@@ -140,17 +138,30 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
140138
})
141139
}
142140

143-
test("Run SparkPi using the remote example jar with a test secret mounted into the driver and " +
144-
"executor pods") {
141+
test("Run PageRank using remote data file") {
145142
sparkAppConf
143+
.set("spark.kubernetes.mountDependencies.filesDownloadDir",
144+
CONTAINER_LOCAL_FILE_DOWNLOAD_PATH)
145+
.set("spark.files", REMOTE_PAGE_RANK_DATA_FILE)
146+
.set("spark.kubernetes.initContainer.image", initContainerImage)
147+
runSparkPageRankAndVerifyCompletion(
148+
appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE))
149+
}
150+
151+
test("Run PageRank using remote data file with test secret mounted into the driver and " +
152+
"executors") {
153+
sparkAppConf
154+
.set("spark.kubernetes.mountDependencies.filesDownloadDir",
155+
CONTAINER_LOCAL_FILE_DOWNLOAD_PATH)
156+
.set("spark.files", REMOTE_PAGE_RANK_DATA_FILE)
146157
.set(s"spark.kubernetes.driver.secrets.$TEST_SECRET_NAME", TEST_SECRET_MOUNT_PATH)
147158
.set(s"spark.kubernetes.executor.secrets.$TEST_SECRET_NAME", TEST_SECRET_MOUNT_PATH)
148-
sparkAppConf.set("spark.kubernetes.initContainer.image", initContainerImage)
159+
.set("spark.kubernetes.initContainer.image", initContainerImage)
149160

150161
createTestSecret()
151162

152-
runSparkPiAndVerifyCompletion(
153-
appResource = REMOTE_EXAMPLES_JAR_URI,
163+
runSparkPageRankAndVerifyCompletion(
164+
appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE),
154165
driverPodChecker = (driverPod: Pod) => {
155166
doBasicDriverPodCheck(driverPod)
156167
checkTestSecret(driverPod, withInitContainer = true)
@@ -161,14 +172,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
161172
})
162173
}
163174

164-
test("Run PageRank using remote data file") {
165-
sparkAppConf
166-
.set("spark.kubernetes.mountDependencies.filesDownloadDir", CONTAINER_LOCAL_FILE_DOWNLOAD_PATH)
167-
.set("spark.files", REMOTE_PAGE_RANK_DATA_FILE)
168-
.set("spark.kubernetes.initContainer.image", initContainerImage)
169-
runSparkPageRankAndVerifyCompletion(appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE))
170-
}
171-
172175
private def runSparkPiAndVerifyCompletion(
173176
appResource: String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR,
174177
driverPodChecker: Pod => Unit = doBasicDriverPodCheck,
@@ -322,9 +325,6 @@ private[spark] object KubernetesSuite {
322325

323326
val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files"
324327

325-
val REMOTE_EXAMPLES_JAR_URI =
326-
"https://storage.googleapis.com/spark-k8s-integration-tests/jars/spark-examples_2.11-2.3.0.jar"
327-
328328
val REMOTE_PAGE_RANK_DATA_FILE =
329329
"https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt"
330330
val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =

0 commit comments

Comments
 (0)