From 3741f9686539c9b4abef4f68c3b5dc1ee176c052 Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Wed, 2 Jul 2025 11:01:54 -0700 Subject: [PATCH 1/2] Fix the bug when user has set the spark.app.id value --- .../org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java index acccb8cd..1a3ebe2e 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java @@ -126,7 +126,7 @@ protected SparkAppDriverConf buildDriverConf( effectiveSparkConf.setIfMissing("spark.app.id", appId); return SparkAppDriverConf.create( effectiveSparkConf, - appId, + effectiveSparkConf.getAppId(), primaryResource, applicationSpec.getMainClass(), applicationSpec.getDriverArgs().toArray(String[]::new), From 394bdb98c6e18b1f33474313ce993ee30461b956 Mon Sep 17 00:00:00 2001 From: Qi Tan <16416018+TQJADE@users.noreply.github.com> Date: Wed, 2 Jul 2025 15:19:49 -0700 Subject: [PATCH 2/2] Add unit test --- .../operator/SparkAppSubmissionWorkerTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java index b41f99a8..b678d94f 100644 --- a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java +++ b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java @@ -219,4 +219,20 @@ void generatedSparkAppIdShouldComplyLengthLimit() { String appId = SparkAppSubmissionWorker.generateSparkAppId(mockApp); assertTrue(appId.length() <= DEFAULT_ID_LENGTH_LIMIT); } + + @Test + void checkAppIdWhenUserSpecifiedInSparkConf() { + SparkApplication mockApp = mock(SparkApplication.class); + ApplicationSpec mockSpec = mock(ApplicationSpec.class); + Map appProps = new HashMap<>(); + appProps.put("spark.app.id", "foo"); + ObjectMeta appMeta = new ObjectMetaBuilder().withName("app1").withNamespace("ns1").build(); + when(mockSpec.getSparkConf()).thenReturn(appProps); + when(mockApp.getSpec()).thenReturn(mockSpec); + when(mockApp.getMetadata()).thenReturn(appMeta); + + SparkAppSubmissionWorker submissionWorker = new SparkAppSubmissionWorker(); + SparkAppDriverConf conf = submissionWorker.buildDriverConf(mockApp, Collections.emptyMap()); + assertEquals(conf.appId(), "foo"); + } }