[SPARK-52606] Promote Spark CRDs to v1

dongjoon-hyun · dongjoon-hyun · commit 39b04c4eff64 · 2025-06-28T13:50:04.000-07:00
### What changes were proposed in this pull request? This PR aims to promote `Spark CRDs` from `v1beta1` to `v1` at Apache Spark K8s Operator `0.5.0`. After finalizing CRDs to `v1`, we are going to start to prepare `Apache Spark K8s Operator 1.0` release. ### Why are the changes needed? To show the maturity of these CRDs according to K8s API versioning. - https://kubernetes.io/docs/reference/using-api/ | Version | Description | | ------- | ----------- | | Alpha | The software is recommended for use only in short-lived testing clusters, <br> due to increased risk of bugs and lack of long-term support | | Beta | The software is not recommended for production uses. <br> The support for a feature will not be dropped, though the details may change. | | Stable | The version name is vX where X is an integer | ### Does this PR introduce _any_ user-facing change? We support all `v1alpha`, `v1beta1`, and `v1` because they are identical. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes apache#261 from dongjoon-hyun/SPARK-52606. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
diff --git a/build-tools/helm/spark-kubernetes-operator/README.md b/build-tools/helm/spark-kubernetes-operator/README.md
@@ -40,6 +40,6 @@ cluster using the [Helm](https://helm.sh) package manager. With this, you can la
 
 - Support Apache Spark 3.5+
 - Support `SparkApp` and `SparkCluster` CRDs
-  - `sparkapplications.spark.apache.org` (v1beta1)
-  - `sparkclusters.spark.apache.org` (v1beta1)
+  - `sparkapplications.spark.apache.org` (v1)
+  - `sparkclusters.spark.apache.org` (v1)
 - Support HPA for SparkCluster
diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml
diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml
diff --git a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml
@@ -161,7 +161,7 @@ metadata:
 {{- end }}
 {{- if $workloadResources.sparkApplicationSentinel.create }}
 {{- range $sentinelNs := .Values.workloadResources.sparkApplicationSentinel.sentinelNamespaces.data }}
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: {{ $workloadResources.sparkApplicationSentinel.name }}
diff --git a/docs/operations.md b/docs/operations.md
@@ -144,7 +144,7 @@ send the HTTP Get to the liveness endpoint, and the
 kubelet will then kill the spark operator container and restart it.
 
 ```yaml
-apiVersion: org.apache.spark/v1beta1
+apiVersion: org.apache.spark/v1
 kind: SparkApplication
 metadata:
   name: spark-sentinel-resources
diff --git a/docs/spark_custom_resources.md b/docs/spark_custom_resources.md
@@ -35,7 +35,7 @@ SparkApplication can be defined in YAML format. User may configure the applicati
 and configurations. Let's start with the [Spark-Pi example](../examples/pi.yaml):
 
 ```yaml
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi
diff --git a/examples/cluster-java21.yaml b/examples/cluster-java21.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-java21
diff --git a/examples/cluster-on-yunikorn.yaml b/examples/cluster-on-yunikorn.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-on-yunikorn
diff --git a/examples/cluster-with-hpa-template.yaml b/examples/cluster-with-hpa-template.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa-template
diff --git a/examples/cluster-with-hpa.yaml b/examples/cluster-with-hpa.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa
diff --git a/examples/cluster-with-template.yaml b/examples/cluster-with-template.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-template
diff --git a/examples/dfs-read-write.yaml b/examples/dfs-read-write.yaml
@@ -15,7 +15,7 @@
 #
 # Since this requires a remote storage, prepare it via `localstack.yml`.
 #
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: dfs-read-write
diff --git a/examples/pi-java21.yaml b/examples/pi-java21.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-java21
diff --git a/examples/pi-on-yunikorn.yaml b/examples/pi-on-yunikorn.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-on-yunikorn
diff --git a/examples/pi-scala.yaml b/examples/pi-scala.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-scala
diff --git a/examples/pi-with-driver-timeout.yaml b/examples/pi-with-driver-timeout.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-driver-timeout
diff --git a/examples/pi-with-eventlog.yaml b/examples/pi-with-eventlog.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-eventlog
diff --git a/examples/pi-with-one-pod.yaml b/examples/pi-with-one-pod.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-one-pod
diff --git a/examples/pi-with-spark-connect-plugin.yaml b/examples/pi-with-spark-connect-plugin.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-spark-connect-plugin
diff --git a/examples/pi.yaml b/examples/pi.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi
diff --git a/examples/prod-cluster-with-three-workers.yaml b/examples/prod-cluster-with-three-workers.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: prod
diff --git a/examples/pyspark-pi.yaml b/examples/pyspark-pi.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-python
diff --git a/examples/qa-cluster-with-one-worker.yaml b/examples/qa-cluster-with-one-worker.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: qa
diff --git a/examples/spark-connect-server-with-spark-cluster.yaml b/examples/spark-connect-server-with-spark-cluster.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-connect-server
diff --git a/examples/spark-connect-server.yaml b/examples/spark-connect-server.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-connect-server
diff --git a/examples/spark-history-server.yaml b/examples/spark-history-server.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-history-server
diff --git a/examples/spark-thrift-server.yaml b/examples/spark-thrift-server.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-thrift-server
diff --git a/examples/sql.yaml b/examples/sql.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: sql
diff --git a/examples/stream-word-count.yaml b/examples/stream-word-count.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: stream-word-count
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java
@@ -23,7 +23,7 @@
 @SuppressWarnings("PMD.DataClass")
 public class Constants {
   public static final String API_GROUP = "spark.apache.org";
-  public static final String API_VERSION = "v1beta1";
+  public static final String API_VERSION = "v1";
   public static final String LABEL_SPARK_APPLICATION_NAME = "spark.operator/spark-app-name";
   public static final String LABEL_SPARK_CLUSTER_NAME = "spark.operator/spark-cluster-name";
   public static final String LABEL_SPARK_OPERATOR_NAME = "spark.operator/name";
diff --git a/tests/e2e/assertions/spark-application/spark-state-transition.yaml b/tests/e2e/assertions/spark-application/spark-state-transition.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
diff --git a/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml b/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test
diff --git a/tests/e2e/python/chainsaw-test.yaml b/tests/e2e/python/chainsaw-test.yaml
@@ -54,7 +54,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1beta1
+            apiVersion: spark.apache.org/v1
             kind: SparkApplication
             namespace: default
         - describe:
diff --git a/tests/e2e/python/pyspark-example.yaml b/tests/e2e/python/pyspark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)
diff --git a/tests/e2e/spark-versions/chainsaw-test.yaml b/tests/e2e/spark-versions/chainsaw-test.yaml
@@ -83,7 +83,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1beta1
+            apiVersion: spark.apache.org/v1
             kind: SparkApplication
             namespace: default
         - describe:
diff --git a/tests/e2e/spark-versions/spark-example.yaml b/tests/e2e/spark-versions/spark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)
diff --git a/tests/e2e/state-transition/chainsaw-test.yaml b/tests/e2e/state-transition/chainsaw-test.yaml
@@ -47,7 +47,7 @@ spec:
         file: "../assertions/spark-application/spark-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1beta1
+        apiVersion: spark.apache.org/v1
         kind: SparkApplication
         namespace: default
     finally:
@@ -72,7 +72,7 @@ spec:
         file: "../assertions/spark-cluster/spark-cluster-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1beta1
+        apiVersion: spark.apache.org/v1
         kind: SparkCluster
         namespace: default
     finally:
diff --git a/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml b/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml
@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test
diff --git a/tests/e2e/state-transition/spark-example-succeeded.yaml b/tests/e2e/state-transition/spark-example-succeeded.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
diff --git a/tests/e2e/watched-namespaces/chainsaw-test.yaml b/tests/e2e/watched-namespaces/chainsaw-test.yaml
@@ -59,11 +59,11 @@ spec:
           namespace: default
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-1
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-2
     finally:
@@ -103,7 +103,7 @@ spec:
           namespace: default-2
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-3
       - get:
diff --git a/tests/e2e/watched-namespaces/spark-example.yaml b/tests/e2e/watched-namespaces/spark-example.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test
