[SPARK-52250] Promote Spark CRDs to v1beta1

dongjoon-hyun · dongjoon-hyun · commit add99bbe7246 · 2025-05-21T17:31:14.000-07:00
### What changes were proposed in this pull request? In line with Apache Spark 4.0.0 release, this PR aims to promote `Spark CRDs` from `v1alpha1` to `v1beta1` at Apache Spark K8s Operator `0.3.0`. ### Why are the changes needed? To show the maturity of these CRDs according to K8s API versioning. - https://kubernetes.io/docs/reference/using-api/ | Version | Description | | ------- | ----------- | | Alpha | The software is recommended for use only in short-lived testing clusters, <br> due to increased risk of bugs and lack of long-term support | | Beta | The software is not recommended for production uses. <br> The support for a feature will not be dropped, though the details may change. | | Stable | The version name is vX where X is an integer | ### Does this PR introduce _any_ user-facing change? Yes, the existing CRDs should be removed cleanly before installing `v1beta1`. However, we are still in `0.3.0`. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #219 from dongjoon-hyun/SPARK-52250. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
diff --git a/build-tools/helm/spark-kubernetes-operator/README.md b/build-tools/helm/spark-kubernetes-operator/README.md
@@ -40,6 +40,6 @@ cluster using the [Helm](https://helm.sh) package manager. With this, you can la
 
 - Support Apache Spark 3.5+
 - Support `SparkApp` and `SparkCluster` CRDs
-  - `sparkapplications.spark.apache.org` (v1alpha1)
-  - `sparkclusters.spark.apache.org` (v1alpha1)
+  - `sparkapplications.spark.apache.org` (v1beta1)
+  - `sparkclusters.spark.apache.org` (v1beta1)
 - Support HPA for SparkCluster
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml
@@ -161,7 +161,7 @@ metadata:
 {{- end }}
 {{- if $workloadResources.sparkApplicationSentinel.create }}
 {{- range $sentinelNs := .Values.workloadResources.sparkApplicationSentinel.sentinelNamespaces.data }}
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: {{ $workloadResources.sparkApplicationSentinel.name }}
diff --git a/docs/operations.md b/docs/operations.md
@@ -142,7 +142,7 @@ send the HTTP Get to the liveness endpoint, and the
 kubelet will then kill the spark operator container and restart it.
 
 ```yaml
-apiVersion: org.apache.spark/v1alpha1
+apiVersion: org.apache.spark/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-sentinel-resources
diff --git a/docs/spark_custom_resources.md b/docs/spark_custom_resources.md
@@ -35,7 +35,7 @@ SparkApplication can be defined in YAML format. User may configure the applicati
 and configurations. Let's start with the [Spark-Pi example](../examples/pi.yaml):
 
 ```yaml
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi
diff --git a/examples/cluster-java21.yaml b/examples/cluster-java21.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: cluster-java21
diff --git a/examples/cluster-on-yunikorn.yaml b/examples/cluster-on-yunikorn.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: cluster-on-yunikorn
diff --git a/examples/cluster-with-hpa-template.yaml b/examples/cluster-with-hpa-template.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa-template
diff --git a/examples/cluster-with-hpa.yaml b/examples/cluster-with-hpa.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa
diff --git a/examples/cluster-with-template.yaml b/examples/cluster-with-template.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: cluster-with-template
diff --git a/examples/pi-java21.yaml b/examples/pi-java21.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-java21
diff --git a/examples/pi-on-yunikorn.yaml b/examples/pi-on-yunikorn.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-on-yunikorn
diff --git a/examples/pi-scala.yaml b/examples/pi-scala.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-scala
diff --git a/examples/pi-with-one-pod.yaml b/examples/pi-with-one-pod.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-with-one-pod
diff --git a/examples/pi-with-spark-connect-plugin.yaml b/examples/pi-with-spark-connect-plugin.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-with-spark-connect-plugin
diff --git a/examples/pi.yaml b/examples/pi.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi
diff --git a/examples/prod-cluster-with-three-workers.yaml b/examples/prod-cluster-with-three-workers.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: prod
diff --git a/examples/pyspark-pi.yaml b/examples/pyspark-pi.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: pi-python
diff --git a/examples/qa-cluster-with-one-worker.yaml b/examples/qa-cluster-with-one-worker.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: qa
diff --git a/examples/spark-connect-server-with-spark-cluster.yaml b/examples/spark-connect-server-with-spark-cluster.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-connect-server
diff --git a/examples/spark-connect-server.yaml b/examples/spark-connect-server.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-connect-server
diff --git a/examples/sql.yaml b/examples/sql.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: sql
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java
@@ -22,7 +22,7 @@
 @SuppressWarnings("PMD.DataClass")
 public class Constants {
   public static final String API_GROUP = "spark.apache.org";
-  public static final String API_VERSION = "v1alpha1";
+  public static final String API_VERSION = "v1beta1";
   public static final String LABEL_SPARK_APPLICATION_NAME = "spark.operator/spark-app-name";
   public static final String LABEL_SPARK_CLUSTER_NAME = "spark.operator/spark-cluster-name";
   public static final String LABEL_SPARK_OPERATOR_NAME = "spark.operator/name";
diff --git a/tests/e2e/assertions/spark-application/spark-state-transition.yaml b/tests/e2e/assertions/spark-application/spark-state-transition.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
diff --git a/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml b/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test
diff --git a/tests/e2e/python/chainsaw-test.yaml b/tests/e2e/python/chainsaw-test.yaml
@@ -54,7 +54,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1alpha1
+            apiVersion: spark.apache.org/v1beta1
             kind: SparkApplication
             namespace: default
         - describe:
diff --git a/tests/e2e/python/pyspark-example.yaml b/tests/e2e/python/pyspark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)
diff --git a/tests/e2e/spark-versions/chainsaw-test.yaml b/tests/e2e/spark-versions/chainsaw-test.yaml
@@ -83,7 +83,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1alpha1
+            apiVersion: spark.apache.org/v1beta1
             kind: SparkApplication
             namespace: default
         - describe:
diff --git a/tests/e2e/spark-versions/spark-example.yaml b/tests/e2e/spark-versions/spark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)
diff --git a/tests/e2e/state-transition/chainsaw-test.yaml b/tests/e2e/state-transition/chainsaw-test.yaml
@@ -47,7 +47,7 @@ spec:
         file: "../assertions/spark-application/spark-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1alpha1
+        apiVersion: spark.apache.org/v1beta1
         kind: SparkApplication
         namespace: default
     finally:
@@ -72,7 +72,7 @@ spec:
         file: "../assertions/spark-cluster/spark-cluster-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1alpha1
+        apiVersion: spark.apache.org/v1beta1
         kind: SparkCluster
         namespace: default
     finally:
diff --git a/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml b/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test
diff --git a/tests/e2e/state-transition/spark-example-succeeded.yaml b/tests/e2e/state-transition/spark-example-succeeded.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
diff --git a/tests/e2e/watched-namespaces/chainsaw-test.yaml b/tests/e2e/watched-namespaces/chainsaw-test.yaml
@@ -59,11 +59,11 @@ spec:
           namespace: default
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1alpha1
+          apiVersion: spark.apache.org/v1beta1
           kind: SparkApplication
           namespace: spark-1
       - describe:
-          apiVersion: spark.apache.org/v1alpha1
+          apiVersion: spark.apache.org/v1beta1
           kind: SparkApplication
           namespace: spark-2
     finally:
@@ -103,7 +103,7 @@ spec:
           namespace: default-2
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1alpha1
+          apiVersion: spark.apache.org/v1beta1
           kind: SparkApplication
           namespace: spark-3
       - get:
@@ -119,4 +119,4 @@ spec:
             kubectl delete sparkapplication spark-job-succeeded-test -n spark-3 --ignore-not-found=true
             helm uninstall spark-kubernetes-operator -n default-2
             kubectl delete namespace default-2 --ignore-not-found=true
-            kubectl delete namespace spark-3 --ignore-not-found=true
+            kubectl delete namespace spark-3 --ignore-not-found=true
diff --git a/tests/e2e/watched-namespaces/spark-example.yaml b/tests/e2e/watched-namespaces/spark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1alpha1
+apiVersion: spark.apache.org/v1beta1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
