[SPARK-53703] Improve unit tests to be neutral from Spark versions

### What changes were proposed in this pull request?

This PR aims to improve unit tests to be neutral from Spark versions.

### Why are the changes needed?

It's important to become robust for (1) the future development and (2) the downstream fork's version changes.

### Does this PR introduce _any_ user-facing change?

No. This is a test-only change.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#339 from dongjoon-hyun/SPARK-53703.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: dongjoon-hyun

spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json

@@ -1,15 +1,6 @@
 {
-  "mainClass": "org.apache.spark.examples.SparkPi",
-  "jars": "local:///opt/spark/examples/jars/spark-examples.jar",
-  "sparkConf": {
-    "spark.kubernetes.driver.master": "local[10]",
-    "spark.kubernetes.driver.request.cores": "5",
-    "spark.kubernetes.driver.limit.cores": "5",
-    "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
-    "spark.kubernetes.container.image": "apache/spark:4.0.0"
-  },
   "runtimeVersions": {
-    "sparkVersion": "4.0.0"
+    "sparkVersion": "test"
   },
   "driverServiceIngressList": [
     {
@@ -56,4 +47,4 @@
       }
     }
   ]
-}
+}

spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java

@@ -58,6 +58,7 @@ class SparkClusterResourceSpecTest {
   RuntimeVersions runtimeVersions = new RuntimeVersions();
   SparkConf sparkConf = new SparkConf().set("spark.kubernetes.namespace", "other-namespace");
   ClusterTolerations clusterTolerations = new ClusterTolerations();
+  static final String VERSION = "dev";
 
   @BeforeEach
   void setUp() {
@@ -76,7 +77,7 @@ void setUp() {
     when(clusterSpec.getMasterSpec()).thenReturn(masterSpec);
     when(clusterSpec.getWorkerSpec()).thenReturn(workerSpec);
     when(clusterSpec.getRuntimeVersions()).thenReturn(runtimeVersions);
-    runtimeVersions.setSparkVersion("4.0.0");
+    runtimeVersions.setSparkVersion(VERSION);
     when(masterSpec.getStatefulSetSpec()).thenReturn(statefulSetSpec);
     when(masterSpec.getStatefulSetMetadata()).thenReturn(objectMeta);
     when(masterSpec.getServiceSpec()).thenReturn(serviceSpec);
@@ -92,19 +93,19 @@ void testMasterService() {
     Service service1 = new SparkClusterResourceSpec(cluster, new SparkConf()).getMasterService();
     assertEquals("my-namespace", service1.getMetadata().getNamespace());
     assertEquals("cluster-name-master-svc", service1.getMetadata().getName());
-    assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
 
     Service service2 = new SparkClusterResourceSpec(cluster, sparkConf).getMasterService();
     assertEquals("other-namespace", service2.getMetadata().getNamespace());
-    assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
   }
 
   @Test
   void testWorkerService() {
     Service service1 = new SparkClusterResourceSpec(cluster, new SparkConf()).getWorkerService();
     assertEquals("my-namespace", service1.getMetadata().getNamespace());
     assertEquals("cluster-name-worker-svc", service1.getMetadata().getName());
-    assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
 
     Service service2 = new SparkClusterResourceSpec(cluster, sparkConf).getMasterService();
     assertEquals("other-namespace", service2.getMetadata().getNamespace());
@@ -128,7 +129,7 @@ void testWorkerServiceWithTemplate() {
     assertEquals("my-namespace", service1.getMetadata().getNamespace());
     assertEquals("cluster-name-worker-svc", service1.getMetadata().getName());
     assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
-    assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals("foo", service1.getSpec().getExternalName());
     assertEquals(
         Map.of(
@@ -157,7 +158,7 @@ void testMasterServiceWithTemplate() {
     assertEquals("my-namespace", service1.getMetadata().getNamespace());
     assertEquals("cluster-name-master-svc", service1.getMetadata().getName());
     assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
-    assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals("foo", service1.getSpec().getExternalName());
     assertEquals(
         Map.of(
@@ -174,9 +175,9 @@ void testMasterStatefulSet() {
     StatefulSet statefulSet1 = spec1.getMasterStatefulSet();
     assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace());
     assertEquals("cluster-name-master", statefulSet1.getMetadata().getName());
-    assertEquals("4.0.0", statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals(
-        "4.0.0",
+        VERSION,
         statefulSet1
             .getSpec()
             .getTemplate()
@@ -230,11 +231,11 @@ void testMasterStatefulSetWithTemplate() {
     assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace());
     assertEquals("cluster-name-master", statefulSet1.getMetadata().getName());
     assertEquals("bar", statefulSet1.getMetadata().getLabels().get("foo"));
-    assertEquals("4.0.0", statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals(1, statefulSet1.getSpec().getTemplate().getSpec().getInitContainers().size());
     assertEquals(2, statefulSet1.getSpec().getTemplate().getSpec().getContainers().size());
     assertEquals(
-        "4.0.0",
+        VERSION,
         statefulSet1
             .getSpec()
             .getTemplate()
@@ -249,9 +250,9 @@ void testWorkerStatefulSet() {
     StatefulSet statefulSet = spec.getWorkerStatefulSet();
     assertEquals("my-namespace", statefulSet.getMetadata().getNamespace());
     assertEquals("cluster-name-worker", statefulSet.getMetadata().getName());
-    assertEquals("4.0.0", statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals(
-        "4.0.0",
+        VERSION,
         statefulSet
             .getSpec()
             .getTemplate()
@@ -304,9 +305,9 @@ void testWorkerStatefulSetWithTemplate() {
     StatefulSet statefulSet = spec.getWorkerStatefulSet();
     assertEquals("my-namespace", statefulSet.getMetadata().getNamespace());
     assertEquals("cluster-name-worker", statefulSet.getMetadata().getName());
-    assertEquals("4.0.0", statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals(
-        "4.0.0",
+        VERSION,
         statefulSet
             .getSpec()
             .getTemplate()
@@ -338,7 +339,7 @@ void testHorizontalPodAutoscaler() {
     assertEquals("HorizontalPodAutoscaler", hpa.getKind());
     assertEquals("my-namespace", hpa.getMetadata().getNamespace());
     assertEquals("cluster-name-worker-hpa", hpa.getMetadata().getName());
-    assertEquals("4.0.0", hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+    assertEquals(VERSION, hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
     assertEquals(1, hpa.getSpec().getMinReplicas());
     assertEquals(3, hpa.getSpec().getMaxReplicas());
     assertEquals(1, hpa.getSpec().getMetrics().size());