[SPARK-52606] Promote Spark CRDs to v1

build-tools/helm/spark-kubernetes-operator/README.md

@@ -40,6 +40,6 @@ cluster using the [Helm](https://helm.sh) package manager. With this, you can la
 
 - Support Apache Spark 3.5+
 - Support `SparkApp` and `SparkCluster` CRDs
-  - `sparkapplications.spark.apache.org` (v1beta1)
-  - `sparkclusters.spark.apache.org` (v1beta1)
+  - `sparkapplications.spark.apache.org` (v1)
+  - `sparkclusters.spark.apache.org` (v1)
 - Support HPA for SparkCluster

build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml

@@ -161,7 +161,7 @@ metadata:
 {{- end }}
 {{- if $workloadResources.sparkApplicationSentinel.create }}
 {{- range $sentinelNs := .Values.workloadResources.sparkApplicationSentinel.sentinelNamespaces.data }}
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: {{ $workloadResources.sparkApplicationSentinel.name }}

docs/operations.md

@@ -144,7 +144,7 @@ send the HTTP Get to the liveness endpoint, and the
 kubelet will then kill the spark operator container and restart it.
 
 ```yaml
-apiVersion: org.apache.spark/v1beta1
+apiVersion: org.apache.spark/v1
 kind: SparkApplication
 metadata:
   name: spark-sentinel-resources

docs/spark_custom_resources.md

@@ -35,7 +35,7 @@ SparkApplication can be defined in YAML format. User may configure the applicati
 and configurations. Let's start with the [Spark-Pi example](../examples/pi.yaml):
 
 ```yaml
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi

examples/cluster-java21.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-java21

examples/cluster-on-yunikorn.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-on-yunikorn

examples/cluster-with-hpa-template.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa-template

examples/cluster-with-hpa.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-hpa

examples/cluster-with-template.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: cluster-with-template

examples/dfs-read-write.yaml

@@ -15,7 +15,7 @@
 #
 # Since this requires a remote storage, prepare it via `localstack.yml`.
 #
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: dfs-read-write

examples/pi-java21.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-java21

examples/pi-on-yunikorn.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-on-yunikorn

examples/pi-scala.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-scala

examples/pi-with-driver-timeout.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-driver-timeout

examples/pi-with-eventlog.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-eventlog

examples/pi-with-one-pod.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-one-pod

examples/pi-with-spark-connect-plugin.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-with-spark-connect-plugin

examples/pi.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi

examples/prod-cluster-with-three-workers.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: prod

examples/pyspark-pi.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: pi-python

examples/qa-cluster-with-one-worker.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: qa

examples/spark-connect-server-with-spark-cluster.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-connect-server

examples/spark-connect-server.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-connect-server

examples/spark-history-server.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-history-server

examples/spark-thrift-server.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-thrift-server

examples/sql.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: sql

examples/stream-word-count.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: stream-word-count

spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java

@@ -22,7 +22,7 @@
 @SuppressWarnings("PMD.DataClass")
 public class Constants {
   public static final String API_GROUP = "spark.apache.org";
-  public static final String API_VERSION = "v1beta1";
+  public static final String API_VERSION = "v1";
   public static final String LABEL_SPARK_APPLICATION_NAME = "spark.operator/spark-app-name";
   public static final String LABEL_SPARK_CLUSTER_NAME = "spark.operator/spark-cluster-name";
   public static final String LABEL_SPARK_OPERATOR_NAME = "spark.operator/name";

tests/e2e/assertions/spark-application/spark-state-transition.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test

tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test

tests/e2e/python/chainsaw-test.yaml

@@ -54,7 +54,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1beta1
+            apiVersion: spark.apache.org/v1
             kind: SparkApplication
             namespace: default
         - describe:

tests/e2e/python/pyspark-example.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)

tests/e2e/spark-versions/chainsaw-test.yaml

@@ -83,7 +83,7 @@ spec:
         - events:
             namespace: default
         - describe:
-            apiVersion: spark.apache.org/v1beta1
+            apiVersion: spark.apache.org/v1
             kind: SparkApplication
             namespace: default
         - describe:

tests/e2e/spark-versions/spark-example.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: ($SPARK_APPLICATION_NAME)

tests/e2e/state-transition/chainsaw-test.yaml

@@ -47,7 +47,7 @@ spec:
         file: "../assertions/spark-application/spark-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1beta1
+        apiVersion: spark.apache.org/v1
         kind: SparkApplication
         namespace: default
     finally:
@@ -72,7 +72,7 @@ spec:
         file: "../assertions/spark-cluster/spark-cluster-state-transition.yaml"
     catch:
     - describe:
-        apiVersion: spark.apache.org/v1beta1
+        apiVersion: spark.apache.org/v1
         kind: SparkCluster
         namespace: default
     finally:

tests/e2e/state-transition/spark-cluster-example-succeeded.yaml

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkCluster
 metadata:
   name: spark-cluster-succeeded-test

tests/e2e/state-transition/spark-example-succeeded.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test

tests/e2e/watched-namespaces/chainsaw-test.yaml

@@ -59,11 +59,11 @@ spec:
           namespace: default
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-1
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-2
     finally:
@@ -103,7 +103,7 @@ spec:
           namespace: default-2
           selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator
       - describe:
-          apiVersion: spark.apache.org/v1beta1
+          apiVersion: spark.apache.org/v1
           kind: SparkApplication
           namespace: spark-3
       - get:

tests/e2e/watched-namespaces/spark-example.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-apiVersion: spark.apache.org/v1beta1
+apiVersion: spark.apache.org/v1
 kind: SparkApplication
 metadata:
   name: spark-job-succeeded-test