From e8874f40a258a26e95da468eff85d94d9f14bd9b Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 21 May 2025 15:57:01 -0700 Subject: [PATCH] v1beta1 test --- build-tools/helm/spark-kubernetes-operator/README.md | 4 ++-- .../templates/workload-rbac.yaml | 2 +- docs/operations.md | 2 +- docs/spark_custom_resources.md | 2 +- examples/cluster-java21.yaml | 2 +- examples/cluster-on-yunikorn.yaml | 2 +- examples/cluster-with-hpa-template.yaml | 2 +- examples/cluster-with-hpa.yaml | 2 +- examples/cluster-with-template.yaml | 2 +- examples/pi-java21.yaml | 2 +- examples/pi-on-yunikorn.yaml | 2 +- examples/pi-scala.yaml | 2 +- examples/pi-with-one-pod.yaml | 2 +- examples/pi-with-spark-connect-plugin.yaml | 2 +- examples/pi.yaml | 2 +- examples/prod-cluster-with-three-workers.yaml | 2 +- examples/pyspark-pi.yaml | 2 +- examples/qa-cluster-with-one-worker.yaml | 2 +- examples/spark-connect-server-with-spark-cluster.yaml | 2 +- examples/spark-connect-server.yaml | 2 +- examples/sql.yaml | 2 +- .../java/org/apache/spark/k8s/operator/Constants.java | 2 +- .../spark-application/spark-state-transition.yaml | 2 +- .../spark-cluster/spark-cluster-state-transition.yaml | 2 +- tests/e2e/python/chainsaw-test.yaml | 2 +- tests/e2e/python/pyspark-example.yaml | 2 +- tests/e2e/spark-versions/chainsaw-test.yaml | 2 +- tests/e2e/spark-versions/spark-example.yaml | 2 +- tests/e2e/state-transition/chainsaw-test.yaml | 4 ++-- .../state-transition/spark-cluster-example-succeeded.yaml | 2 +- tests/e2e/state-transition/spark-example-succeeded.yaml | 2 +- tests/e2e/watched-namespaces/chainsaw-test.yaml | 8 ++++---- tests/e2e/watched-namespaces/spark-example.yaml | 2 +- 33 files changed, 38 insertions(+), 38 deletions(-) diff --git a/build-tools/helm/spark-kubernetes-operator/README.md b/build-tools/helm/spark-kubernetes-operator/README.md index d15b0a31..0514514f 100644 --- a/build-tools/helm/spark-kubernetes-operator/README.md +++ b/build-tools/helm/spark-kubernetes-operator/README.md @@ -40,6 +40,6 @@ cluster using the [Helm](https://helm.sh) package manager. With this, you can la - Support Apache Spark 3.5+ - Support `SparkApp` and `SparkCluster` CRDs - - `sparkapplications.spark.apache.org` (v1alpha1) - - `sparkclusters.spark.apache.org` (v1alpha1) + - `sparkapplications.spark.apache.org` (v1beta1) + - `sparkclusters.spark.apache.org` (v1beta1) - Support HPA for SparkCluster diff --git a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml index 06ed7880..89bb509a 100644 --- a/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml +++ b/build-tools/helm/spark-kubernetes-operator/templates/workload-rbac.yaml @@ -161,7 +161,7 @@ metadata: {{- end }} {{- if $workloadResources.sparkApplicationSentinel.create }} {{- range $sentinelNs := .Values.workloadResources.sparkApplicationSentinel.sentinelNamespaces.data }} -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: {{ $workloadResources.sparkApplicationSentinel.name }} diff --git a/docs/operations.md b/docs/operations.md index ca93b00a..80fba145 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -142,7 +142,7 @@ send the HTTP Get to the liveness endpoint, and the kubelet will then kill the spark operator container and restart it. ```yaml -apiVersion: org.apache.spark/v1alpha1 +apiVersion: org.apache.spark/v1beta1 kind: SparkApplication metadata: name: spark-sentinel-resources diff --git a/docs/spark_custom_resources.md b/docs/spark_custom_resources.md index 6a8751d4..8b5dbab5 100644 --- a/docs/spark_custom_resources.md +++ b/docs/spark_custom_resources.md @@ -35,7 +35,7 @@ SparkApplication can be defined in YAML format. User may configure the applicati and configurations. Let's start with the [Spark-Pi example](../examples/pi.yaml): ```yaml -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi diff --git a/examples/cluster-java21.yaml b/examples/cluster-java21.yaml index abc48263..4fe59851 100644 --- a/examples/cluster-java21.yaml +++ b/examples/cluster-java21.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: cluster-java21 diff --git a/examples/cluster-on-yunikorn.yaml b/examples/cluster-on-yunikorn.yaml index 41a0f4a4..75f27fe2 100644 --- a/examples/cluster-on-yunikorn.yaml +++ b/examples/cluster-on-yunikorn.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: cluster-on-yunikorn diff --git a/examples/cluster-with-hpa-template.yaml b/examples/cluster-with-hpa-template.yaml index 5220690e..d251fe24 100644 --- a/examples/cluster-with-hpa-template.yaml +++ b/examples/cluster-with-hpa-template.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: cluster-with-hpa-template diff --git a/examples/cluster-with-hpa.yaml b/examples/cluster-with-hpa.yaml index ca820881..93c19d0b 100644 --- a/examples/cluster-with-hpa.yaml +++ b/examples/cluster-with-hpa.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: cluster-with-hpa diff --git a/examples/cluster-with-template.yaml b/examples/cluster-with-template.yaml index 12095003..d4f03398 100644 --- a/examples/cluster-with-template.yaml +++ b/examples/cluster-with-template.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: cluster-with-template diff --git a/examples/pi-java21.yaml b/examples/pi-java21.yaml index a0015a0f..2601d986 100644 --- a/examples/pi-java21.yaml +++ b/examples/pi-java21.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-java21 diff --git a/examples/pi-on-yunikorn.yaml b/examples/pi-on-yunikorn.yaml index 8fda63ae..eb443bc2 100644 --- a/examples/pi-on-yunikorn.yaml +++ b/examples/pi-on-yunikorn.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-on-yunikorn diff --git a/examples/pi-scala.yaml b/examples/pi-scala.yaml index 60418223..5d0a6050 100644 --- a/examples/pi-scala.yaml +++ b/examples/pi-scala.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-scala diff --git a/examples/pi-with-one-pod.yaml b/examples/pi-with-one-pod.yaml index a83560e6..f2646c3d 100644 --- a/examples/pi-with-one-pod.yaml +++ b/examples/pi-with-one-pod.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-with-one-pod diff --git a/examples/pi-with-spark-connect-plugin.yaml b/examples/pi-with-spark-connect-plugin.yaml index 0ab676f3..d1d8fc7d 100644 --- a/examples/pi-with-spark-connect-plugin.yaml +++ b/examples/pi-with-spark-connect-plugin.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-with-spark-connect-plugin diff --git a/examples/pi.yaml b/examples/pi.yaml index bfcd73dc..1eefadbf 100644 --- a/examples/pi.yaml +++ b/examples/pi.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi diff --git a/examples/prod-cluster-with-three-workers.yaml b/examples/prod-cluster-with-three-workers.yaml index ffcd41da..eb884cc6 100644 --- a/examples/prod-cluster-with-three-workers.yaml +++ b/examples/prod-cluster-with-three-workers.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: prod diff --git a/examples/pyspark-pi.yaml b/examples/pyspark-pi.yaml index 7c9f46d4..db1a25a4 100644 --- a/examples/pyspark-pi.yaml +++ b/examples/pyspark-pi.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: pi-python diff --git a/examples/qa-cluster-with-one-worker.yaml b/examples/qa-cluster-with-one-worker.yaml index 30bc18c6..072e06bb 100644 --- a/examples/qa-cluster-with-one-worker.yaml +++ b/examples/qa-cluster-with-one-worker.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: qa diff --git a/examples/spark-connect-server-with-spark-cluster.yaml b/examples/spark-connect-server-with-spark-cluster.yaml index a26ff3d6..85e75199 100644 --- a/examples/spark-connect-server-with-spark-cluster.yaml +++ b/examples/spark-connect-server-with-spark-cluster.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: spark-connect-server diff --git a/examples/spark-connect-server.yaml b/examples/spark-connect-server.yaml index 23994a30..8efad098 100644 --- a/examples/spark-connect-server.yaml +++ b/examples/spark-connect-server.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: spark-connect-server diff --git a/examples/sql.yaml b/examples/sql.yaml index 0f5cce6c..d9f93667 100644 --- a/examples/sql.yaml +++ b/examples/sql.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: sql diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java index 51b81238..c300041a 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java @@ -22,7 +22,7 @@ @SuppressWarnings("PMD.DataClass") public class Constants { public static final String API_GROUP = "spark.apache.org"; - public static final String API_VERSION = "v1alpha1"; + public static final String API_VERSION = "v1beta1"; public static final String LABEL_SPARK_APPLICATION_NAME = "spark.operator/spark-app-name"; public static final String LABEL_SPARK_CLUSTER_NAME = "spark.operator/spark-cluster-name"; public static final String LABEL_SPARK_OPERATOR_NAME = "spark.operator/name"; diff --git a/tests/e2e/assertions/spark-application/spark-state-transition.yaml b/tests/e2e/assertions/spark-application/spark-state-transition.yaml index c28f7a63..297f6411 100644 --- a/tests/e2e/assertions/spark-application/spark-state-transition.yaml +++ b/tests/e2e/assertions/spark-application/spark-state-transition.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: spark-job-succeeded-test diff --git a/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml b/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml index 4194583a..84df1ab0 100644 --- a/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml +++ b/tests/e2e/assertions/spark-cluster/spark-cluster-state-transition.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: spark-cluster-succeeded-test diff --git a/tests/e2e/python/chainsaw-test.yaml b/tests/e2e/python/chainsaw-test.yaml index 81ab5542..c4d1197c 100644 --- a/tests/e2e/python/chainsaw-test.yaml +++ b/tests/e2e/python/chainsaw-test.yaml @@ -54,7 +54,7 @@ spec: - events: namespace: default - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: default - describe: diff --git a/tests/e2e/python/pyspark-example.yaml b/tests/e2e/python/pyspark-example.yaml index ec5ec3cb..38d63a7c 100644 --- a/tests/e2e/python/pyspark-example.yaml +++ b/tests/e2e/python/pyspark-example.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: ($SPARK_APPLICATION_NAME) diff --git a/tests/e2e/spark-versions/chainsaw-test.yaml b/tests/e2e/spark-versions/chainsaw-test.yaml index 91323bdb..a932eff4 100644 --- a/tests/e2e/spark-versions/chainsaw-test.yaml +++ b/tests/e2e/spark-versions/chainsaw-test.yaml @@ -83,7 +83,7 @@ spec: - events: namespace: default - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: default - describe: diff --git a/tests/e2e/spark-versions/spark-example.yaml b/tests/e2e/spark-versions/spark-example.yaml index 77eeeda8..2fa4f021 100644 --- a/tests/e2e/spark-versions/spark-example.yaml +++ b/tests/e2e/spark-versions/spark-example.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: ($SPARK_APPLICATION_NAME) diff --git a/tests/e2e/state-transition/chainsaw-test.yaml b/tests/e2e/state-transition/chainsaw-test.yaml index 6e95b69a..1ff72c5e 100644 --- a/tests/e2e/state-transition/chainsaw-test.yaml +++ b/tests/e2e/state-transition/chainsaw-test.yaml @@ -47,7 +47,7 @@ spec: file: "../assertions/spark-application/spark-state-transition.yaml" catch: - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: default finally: @@ -72,7 +72,7 @@ spec: file: "../assertions/spark-cluster/spark-cluster-state-transition.yaml" catch: - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkCluster namespace: default finally: diff --git a/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml b/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml index 613acd00..dd6c4a2b 100644 --- a/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml +++ b/tests/e2e/state-transition/spark-cluster-example-succeeded.yaml @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkCluster metadata: name: spark-cluster-succeeded-test diff --git a/tests/e2e/state-transition/spark-example-succeeded.yaml b/tests/e2e/state-transition/spark-example-succeeded.yaml index 9f07a10e..9f573422 100644 --- a/tests/e2e/state-transition/spark-example-succeeded.yaml +++ b/tests/e2e/state-transition/spark-example-succeeded.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: spark-job-succeeded-test diff --git a/tests/e2e/watched-namespaces/chainsaw-test.yaml b/tests/e2e/watched-namespaces/chainsaw-test.yaml index fdffa0a0..a883d15e 100644 --- a/tests/e2e/watched-namespaces/chainsaw-test.yaml +++ b/tests/e2e/watched-namespaces/chainsaw-test.yaml @@ -59,11 +59,11 @@ spec: namespace: default selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: spark-1 - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: spark-2 finally: @@ -103,7 +103,7 @@ spec: namespace: default-2 selector: app.kubernetes.io/component=operator-deployment,app.kubernetes.io/name=spark-kubernetes-operator - describe: - apiVersion: spark.apache.org/v1alpha1 + apiVersion: spark.apache.org/v1beta1 kind: SparkApplication namespace: spark-3 - get: @@ -119,4 +119,4 @@ spec: kubectl delete sparkapplication spark-job-succeeded-test -n spark-3 --ignore-not-found=true helm uninstall spark-kubernetes-operator -n default-2 kubectl delete namespace default-2 --ignore-not-found=true - kubectl delete namespace spark-3 --ignore-not-found=true \ No newline at end of file + kubectl delete namespace spark-3 --ignore-not-found=true diff --git a/tests/e2e/watched-namespaces/spark-example.yaml b/tests/e2e/watched-namespaces/spark-example.yaml index dcc0acba..7d6dc64c 100644 --- a/tests/e2e/watched-namespaces/spark-example.yaml +++ b/tests/e2e/watched-namespaces/spark-example.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -apiVersion: spark.apache.org/v1alpha1 +apiVersion: spark.apache.org/v1beta1 kind: SparkApplication metadata: name: spark-job-succeeded-test