Skip to content

Commit d98a5ce

Browse files
jiangxb1987mengxr
authored andcommitted
[SPARK-28115][CORE][TEST] Fix flaky test: SparkContextSuite.test resource scheduling under local-cluster mode
## What changes were proposed in this pull request? The test `SparkContextSuite.test resource scheduling under local-cluster mode` has been flaky, because it expects the size of `sc.statusTracker.getExecutorInfos` be the same as the number of executors, while the returned list contains both the driver and executors. ## How was this patch tested? Updated existing tests. Closes apache#24917 from jiangxb1987/getExecutorInfos. Authored-by: Xingbo Jiang <[email protected]> Signed-off-by: Xiangrui Meng <[email protected]>
1 parent 25c5d57 commit d98a5ce

File tree

2 files changed

+4
-9
lines changed

2 files changed

+4
-9
lines changed

core/src/main/scala/org/apache/spark/SparkStatusTracker.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ class SparkStatusTracker private[spark] (sc: SparkContext, store: AppStatusStore
9999
/**
100100
* Returns information of all known executors, including host, port, cacheSize, numRunningTasks
101101
* and memory metrics.
102+
* Note this include information for both the driver and executors.
102103
*/
103104
def getExecutorInfos: Array[SparkExecutorInfo] = {
104105
store.executorList(true).map { exec =>

core/src/test/scala/org/apache/spark/SparkContextSuite.scala

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -750,9 +750,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
750750
sc = new SparkContext(conf)
751751

752752
// Ensure all executors has started
753-
eventually(timeout(10.seconds)) {
754-
assert(sc.statusTracker.getExecutorInfos.size == 1)
755-
}
753+
TestUtils.waitUntilExecutorsUp(sc, 1, 10000)
756754
assert(sc.resources.size === 1)
757755
assert(sc.resources.get(GPU).get.addresses === Array("5", "6"))
758756
assert(sc.resources.get(GPU).get.name === "gpu")
@@ -780,9 +778,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
780778
sc = new SparkContext(conf)
781779

782780
// Ensure all executors has started
783-
eventually(timeout(10.seconds)) {
784-
assert(sc.statusTracker.getExecutorInfos.size == 1)
785-
}
781+
TestUtils.waitUntilExecutorsUp(sc, 1, 10000)
786782
// driver gpu resources file should take precedence over the script
787783
assert(sc.resources.size === 1)
788784
assert(sc.resources.get(GPU).get.addresses === Array("0", "1", "8"))
@@ -855,9 +851,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
855851
sc = new SparkContext(conf)
856852

857853
// Ensure all executors has started
858-
eventually(timeout(60.seconds)) {
859-
assert(sc.statusTracker.getExecutorInfos.size == 3)
860-
}
854+
TestUtils.waitUntilExecutorsUp(sc, 3, 60000)
861855

862856
val rdd = sc.makeRDD(1 to 10, 9).mapPartitions { it =>
863857
val context = TaskContext.get()

0 commit comments

Comments
 (0)