[SPARK-52657] Allow spark.app.id

### What changes were proposed in this pull request?
SparkAppDriverConf should use the value stored in effectiveSparkConf rather than pre-computed Id for creation.

### Why are the changes needed?
When user set the spark.app.id in sparkConf, operator should honor this value.

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
unit test

### Was this patch authored or co-authored using generative AI tooling?
no

Closes apache#271 from TQJADE/appid-bug-fix.

Authored-by: Qi Tan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: TQJADE

spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java

@@ -126,7 +126,7 @@ protected SparkAppDriverConf buildDriverConf(
     effectiveSparkConf.setIfMissing("spark.app.id", appId);
     return SparkAppDriverConf.create(
         effectiveSparkConf,
-        appId,
+        effectiveSparkConf.getAppId(),
         primaryResource,
         applicationSpec.getMainClass(),
         applicationSpec.getDriverArgs().toArray(String[]::new),

spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java

@@ -219,4 +219,20 @@ void generatedSparkAppIdShouldComplyLengthLimit() {
     String appId = SparkAppSubmissionWorker.generateSparkAppId(mockApp);
     assertTrue(appId.length() <= DEFAULT_ID_LENGTH_LIMIT);
   }
+
+  @Test
+  void checkAppIdWhenUserSpecifiedInSparkConf() {
+    SparkApplication mockApp = mock(SparkApplication.class);
+    ApplicationSpec mockSpec = mock(ApplicationSpec.class);
+    Map<String, String> appProps = new HashMap<>();
+    appProps.put("spark.app.id", "foo");
+    ObjectMeta appMeta = new ObjectMetaBuilder().withName("app1").withNamespace("ns1").build();
+    when(mockSpec.getSparkConf()).thenReturn(appProps);
+    when(mockApp.getSpec()).thenReturn(mockSpec);
+    when(mockApp.getMetadata()).thenReturn(appMeta);
+
+    SparkAppSubmissionWorker submissionWorker = new SparkAppSubmissionWorker();
+    SparkAppDriverConf conf = submissionWorker.buildDriverConf(mockApp, Collections.emptyMap());
+    assertEquals(conf.appId(), "foo");
+  }
 }