@@ -40,9 +40,12 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
40
40
private var kubernetesTestComponents : KubernetesTestComponents = _
41
41
private var sparkAppConf : SparkAppConf = _
42
42
43
- private val driverImage = System .getProperty(" spark.docker.test.driverImage" , " spark-driver:latest" )
44
- private val executorImage = System .getProperty(" spark.docker.test.executorImage" , " spark-executor:latest" )
45
- private val initContainerImage = System .getProperty(" spark.docker.test.initContainerImage" , " spark-init:latest" )
43
+ private val driverImage = System .getProperty(" spark.docker.test.driverImage" ,
44
+ " spark-driver:latest" )
45
+ private val executorImage = System .getProperty(" spark.docker.test.executorImage" ,
46
+ " spark-executor:latest" )
47
+ private val initContainerImage = System .getProperty(" spark.docker.test.initContainerImage" ,
48
+ " spark-init:latest" )
46
49
47
50
48
51
override def beforeAll (): Unit = {
@@ -91,11 +94,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
91
94
runSparkPiAndVerifyCompletion(appArgs = Array (" 5" ))
92
95
}
93
96
94
- test(" Run SparkPi using the remote example jar." ) {
95
- sparkAppConf.set(" spark.kubernetes.initContainer.image" , initContainerImage)
96
- runSparkPiAndVerifyCompletion(appResource = REMOTE_EXAMPLES_JAR_URI )
97
- }
98
-
99
97
test(" Run SparkPi with custom driver pod name, labels, annotations, and environment variables." ) {
100
98
sparkAppConf
101
99
.set(" spark.kubernetes.driver.pod.name" , " spark-integration-spark-pi" )
@@ -140,17 +138,30 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
140
138
})
141
139
}
142
140
143
- test(" Run SparkPi using the remote example jar with a test secret mounted into the driver and " +
144
- " executor pods" ) {
141
+ test(" Run PageRank using remote data file" ) {
145
142
sparkAppConf
143
+ .set(" spark.kubernetes.mountDependencies.filesDownloadDir" ,
144
+ CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
145
+ .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
146
+ .set(" spark.kubernetes.initContainer.image" , initContainerImage)
147
+ runSparkPageRankAndVerifyCompletion(
148
+ appArgs = Array (CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ))
149
+ }
150
+
151
+ test(" Run PageRank using remote data file with test secret mounted into the driver and " +
152
+ " executors" ) {
153
+ sparkAppConf
154
+ .set(" spark.kubernetes.mountDependencies.filesDownloadDir" ,
155
+ CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
156
+ .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
146
157
.set(s " spark.kubernetes.driver.secrets. $TEST_SECRET_NAME" , TEST_SECRET_MOUNT_PATH )
147
158
.set(s " spark.kubernetes.executor.secrets. $TEST_SECRET_NAME" , TEST_SECRET_MOUNT_PATH )
148
- sparkAppConf .set(" spark.kubernetes.initContainer.image" , initContainerImage)
159
+ .set(" spark.kubernetes.initContainer.image" , initContainerImage)
149
160
150
161
createTestSecret()
151
162
152
- runSparkPiAndVerifyCompletion (
153
- appResource = REMOTE_EXAMPLES_JAR_URI ,
163
+ runSparkPageRankAndVerifyCompletion (
164
+ appArgs = Array ( CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ) ,
154
165
driverPodChecker = (driverPod : Pod ) => {
155
166
doBasicDriverPodCheck(driverPod)
156
167
checkTestSecret(driverPod, withInitContainer = true )
@@ -161,14 +172,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
161
172
})
162
173
}
163
174
164
- test(" Run PageRank using remote data file" ) {
165
- sparkAppConf
166
- .set(" spark.kubernetes.mountDependencies.filesDownloadDir" , CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
167
- .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
168
- .set(" spark.kubernetes.initContainer.image" , initContainerImage)
169
- runSparkPageRankAndVerifyCompletion(appArgs = Array (CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ))
170
- }
171
-
172
175
private def runSparkPiAndVerifyCompletion (
173
176
appResource : String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR ,
174
177
driverPodChecker : Pod => Unit = doBasicDriverPodCheck,
@@ -322,9 +325,6 @@ private[spark] object KubernetesSuite {
322
325
323
326
val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = " /var/spark-data/spark-files"
324
327
325
- val REMOTE_EXAMPLES_JAR_URI =
326
- " https://storage.googleapis.com/spark-k8s-integration-tests/jars/spark-examples_2.11-2.3.0.jar"
327
-
328
328
val REMOTE_PAGE_RANK_DATA_FILE =
329
329
" https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt"
330
330
val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
0 commit comments