Skip to content

Commit 1841dd2

Browse files
committed
[SPARK-53516][CORE][TESTS][FOLLOWUP] Fix compilation errors in SparkPipelinesSuite
### What changes were proposed in this pull request? This pr aim to fix compilation errors in SparkPipelinesSuite ### Why are the changes needed? Restore GA: - https://github.com/apache/spark/actions/runs/17952456210/job/51055288793 ``` [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:123:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:127:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:131:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") == [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:142:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") == [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:153:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") == [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:164:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") == [error] ^ [error] /home/runner/work/spark/spark/core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala:175:63: unknown parameter name: sparkHome [error] SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") == [error] ^ ``` ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GitHub Actions - Manual checked `build/sbt clean "core/testOnly org.apache.spark.deploy.SparkPipelinesSuite"` ``` WARNING: Using incubator modules: jdk.incubator.vector [info] SparkPipelinesSuite: [info] - only spark submit args (20 milliseconds) [info] - only pipelines args (1 millisecond) [info] - spark-submit and pipelines args (0 milliseconds) [info] - class arg prohibited (2 milliseconds) [info] - spark.api.mode arg (1 millisecond) [info] - name arg (0 milliseconds) [info] Run completed in 692 milliseconds. [info] Total number of tests run: 6 [info] Suites: completed 1, aborted 0 [info] Tests: succeeded 6, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. ``` ### Was this patch authored or co-authored using generative AI tooling? No Closes #52422 from LuciferYang/SPARK-53498-FOLLOWUP. Authored-by: yangjie01 <[email protected]> Signed-off-by: yangjie01 <[email protected]>
1 parent c6cea73 commit 1841dd2

File tree

1 file changed

+14
-7
lines changed

1 file changed

+14
-7
lines changed

core/src/test/scala/org/apache/spark/deploy/SparkPipelinesSuite.scala

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -120,15 +120,18 @@ class SparkPipelinesSuite extends SparkSubmitTestUtils with BeforeAndAfterEach {
120120
test("spark.api.mode arg") {
121121
var args = Array("--conf", "spark.api.mode=classic")
122122
intercept[SparkUserAppException] {
123-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc")
123+
SparkPipelines.constructSparkSubmitArgs(
124+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args)
124125
}
125126
args = Array("-c", "spark.api.mode=classic")
126127
intercept[SparkUserAppException] {
127-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc")
128+
SparkPipelines.constructSparkSubmitArgs(
129+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args)
128130
}
129131
args = Array("--conf", "spark.api.mode=CONNECT")
130132
assert(
131-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") ==
133+
SparkPipelines.constructSparkSubmitArgs(
134+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args) ==
132135
Seq(
133136
"--conf",
134137
"spark.api.mode=connect",
@@ -139,7 +142,8 @@ class SparkPipelinesSuite extends SparkSubmitTestUtils with BeforeAndAfterEach {
139142
)
140143
args = Array("--conf", "spark.api.mode=CoNNect")
141144
assert(
142-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") ==
145+
SparkPipelines.constructSparkSubmitArgs(
146+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args) ==
143147
Seq(
144148
"--conf",
145149
"spark.api.mode=connect",
@@ -150,7 +154,8 @@ class SparkPipelinesSuite extends SparkSubmitTestUtils with BeforeAndAfterEach {
150154
)
151155
args = Array("--conf", "spark.api.mode=connect")
152156
assert(
153-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") ==
157+
SparkPipelines.constructSparkSubmitArgs(
158+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args) ==
154159
Seq(
155160
"--conf",
156161
"spark.api.mode=connect",
@@ -161,7 +166,8 @@ class SparkPipelinesSuite extends SparkSubmitTestUtils with BeforeAndAfterEach {
161166
)
162167
args = Array("--conf", "spark.api.mode= connect")
163168
assert(
164-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") ==
169+
SparkPipelines.constructSparkSubmitArgs(
170+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args) ==
165171
Seq(
166172
"--conf",
167173
"spark.api.mode=connect",
@@ -172,7 +178,8 @@ class SparkPipelinesSuite extends SparkSubmitTestUtils with BeforeAndAfterEach {
172178
)
173179
args = Array("-c", "spark.api.mode=connect")
174180
assert(
175-
SparkPipelines.constructSparkSubmitArgs(args, sparkHome = "abc") ==
181+
SparkPipelines.constructSparkSubmitArgs(
182+
pipelinesCliFile = "abc/python/pyspark/pipelines/cli.py", args = args) ==
176183
Seq(
177184
"--conf",
178185
"spark.api.mode=connect",

0 commit comments

Comments
 (0)