diff --git a/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json b/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json index 1f44cbd8..6222fb4d 100644 --- a/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json +++ b/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json @@ -1,15 +1,6 @@ { - "mainClass": "org.apache.spark.examples.SparkPi", - "jars": "local:///opt/spark/examples/jars/spark-examples.jar", - "sparkConf": { - "spark.kubernetes.driver.master": "local[10]", - "spark.kubernetes.driver.request.cores": "5", - "spark.kubernetes.driver.limit.cores": "5", - "spark.kubernetes.authenticate.driver.serviceAccountName": "spark", - "spark.kubernetes.container.image": "apache/spark:4.0.0" - }, "runtimeVersions": { - "sparkVersion": "4.0.0" + "sparkVersion": "test" }, "driverServiceIngressList": [ { @@ -56,4 +47,4 @@ } } ] -} \ No newline at end of file +} diff --git a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java index 7335ff15..9717bf53 100644 --- a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java +++ b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java @@ -58,6 +58,7 @@ class SparkClusterResourceSpecTest { RuntimeVersions runtimeVersions = new RuntimeVersions(); SparkConf sparkConf = new SparkConf().set("spark.kubernetes.namespace", "other-namespace"); ClusterTolerations clusterTolerations = new ClusterTolerations(); + static final String VERSION = "dev"; @BeforeEach void setUp() { @@ -76,7 +77,7 @@ void setUp() { when(clusterSpec.getMasterSpec()).thenReturn(masterSpec); when(clusterSpec.getWorkerSpec()).thenReturn(workerSpec); when(clusterSpec.getRuntimeVersions()).thenReturn(runtimeVersions); - runtimeVersions.setSparkVersion("4.0.0"); + runtimeVersions.setSparkVersion(VERSION); when(masterSpec.getStatefulSetSpec()).thenReturn(statefulSetSpec); when(masterSpec.getStatefulSetMetadata()).thenReturn(objectMeta); when(masterSpec.getServiceSpec()).thenReturn(serviceSpec); @@ -92,11 +93,11 @@ void testMasterService() { Service service1 = new SparkClusterResourceSpec(cluster, new SparkConf()).getMasterService(); assertEquals("my-namespace", service1.getMetadata().getNamespace()); assertEquals("cluster-name-master-svc", service1.getMetadata().getName()); - assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); Service service2 = new SparkClusterResourceSpec(cluster, sparkConf).getMasterService(); assertEquals("other-namespace", service2.getMetadata().getNamespace()); - assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); } @Test @@ -104,7 +105,7 @@ void testWorkerService() { Service service1 = new SparkClusterResourceSpec(cluster, new SparkConf()).getWorkerService(); assertEquals("my-namespace", service1.getMetadata().getNamespace()); assertEquals("cluster-name-worker-svc", service1.getMetadata().getName()); - assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); Service service2 = new SparkClusterResourceSpec(cluster, sparkConf).getMasterService(); assertEquals("other-namespace", service2.getMetadata().getNamespace()); @@ -128,7 +129,7 @@ void testWorkerServiceWithTemplate() { assertEquals("my-namespace", service1.getMetadata().getNamespace()); assertEquals("cluster-name-worker-svc", service1.getMetadata().getName()); assertEquals("bar", service1.getMetadata().getLabels().get("foo")); - assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals("foo", service1.getSpec().getExternalName()); assertEquals( Map.of( @@ -157,7 +158,7 @@ void testMasterServiceWithTemplate() { assertEquals("my-namespace", service1.getMetadata().getNamespace()); assertEquals("cluster-name-master-svc", service1.getMetadata().getName()); assertEquals("bar", service1.getMetadata().getLabels().get("foo")); - assertEquals("4.0.0", service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals("foo", service1.getSpec().getExternalName()); assertEquals( Map.of( @@ -174,9 +175,9 @@ void testMasterStatefulSet() { StatefulSet statefulSet1 = spec1.getMasterStatefulSet(); assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace()); assertEquals("cluster-name-master", statefulSet1.getMetadata().getName()); - assertEquals("4.0.0", statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals( - "4.0.0", + VERSION, statefulSet1 .getSpec() .getTemplate() @@ -230,11 +231,11 @@ void testMasterStatefulSetWithTemplate() { assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace()); assertEquals("cluster-name-master", statefulSet1.getMetadata().getName()); assertEquals("bar", statefulSet1.getMetadata().getLabels().get("foo")); - assertEquals("4.0.0", statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals(1, statefulSet1.getSpec().getTemplate().getSpec().getInitContainers().size()); assertEquals(2, statefulSet1.getSpec().getTemplate().getSpec().getContainers().size()); assertEquals( - "4.0.0", + VERSION, statefulSet1 .getSpec() .getTemplate() @@ -249,9 +250,9 @@ void testWorkerStatefulSet() { StatefulSet statefulSet = spec.getWorkerStatefulSet(); assertEquals("my-namespace", statefulSet.getMetadata().getNamespace()); assertEquals("cluster-name-worker", statefulSet.getMetadata().getName()); - assertEquals("4.0.0", statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals( - "4.0.0", + VERSION, statefulSet .getSpec() .getTemplate() @@ -304,9 +305,9 @@ void testWorkerStatefulSetWithTemplate() { StatefulSet statefulSet = spec.getWorkerStatefulSet(); assertEquals("my-namespace", statefulSet.getMetadata().getNamespace()); assertEquals("cluster-name-worker", statefulSet.getMetadata().getName()); - assertEquals("4.0.0", statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals( - "4.0.0", + VERSION, statefulSet .getSpec() .getTemplate() @@ -338,7 +339,7 @@ void testHorizontalPodAutoscaler() { assertEquals("HorizontalPodAutoscaler", hpa.getKind()); assertEquals("my-namespace", hpa.getMetadata().getNamespace()); assertEquals("cluster-name-worker-hpa", hpa.getMetadata().getName()); - assertEquals("4.0.0", hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); + assertEquals(VERSION, hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals(1, hpa.getSpec().getMinReplicas()); assertEquals(3, hpa.getSpec().getMaxReplicas()); assertEquals(1, hpa.getSpec().getMetrics().size());