From 6ce51896ece5e7ff452ee591b9b5c74ce9f611e9 Mon Sep 17 00:00:00 2001 From: Zhou JIANG Date: Tue, 2 Sep 2025 14:23:27 -0700 Subject: [PATCH 1/3] [SPARK-53457] Expose fabric8 dependencies as api for downstream users ### What changes were proposed in this pull request? This PR exposes fabric8 dependencies as api in gradle build and hence in pom ### Why are the changes needed? Our public API surfaces fabric8 types (e.g. PodTemplateSpec), so hiding the dependencies behind `implementation` may cause comile errors for customers who need to use those types along with our API library. This enhances developer experience and avoid surprises when integrating our API with projects that already depends on fabric8. ### Does this PR introduce _any_ user-facing change? Pom change only ### How was this patch tested? CIs ### Was this patch authored or co-authored using generative AI tooling? No --- spark-operator-api/build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle index 8fe8b7a7..f9323f27 100644 --- a/spark-operator-api/build.gradle +++ b/spark-operator-api/build.gradle @@ -16,9 +16,13 @@ * specific language governing permissions and limitations * under the License. */ +plugins { + id 'java-library' +} + dependencies { // fabric8 - implementation(libs.kubernetes.client) { + api(libs.kubernetes.client) { exclude group: 'com.squareup.okhttp3' } compileOnly(libs.crd.generator.apt) From 8ea6a99364f26def6a8da9534a4ad3da8a81367e Mon Sep 17 00:00:00 2001 From: Zhou JIANG Date: Mon, 15 Sep 2025 17:34:06 -0700 Subject: [PATCH 2/3] Add new library publishing with docs --- docs/spark_operator_api_library.md | 51 +++++++++++++++++++++++++ settings.gradle | 1 + spark-operator-api-fabric8/build.gradle | 30 +++++++++++++++ spark-operator-api/build.gradle | 2 +- 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 docs/spark_operator_api_library.md create mode 100644 spark-operator-api-fabric8/build.gradle diff --git a/docs/spark_operator_api_library.md b/docs/spark_operator_api_library.md new file mode 100644 index 00000000..b6ee2612 --- /dev/null +++ b/docs/spark_operator_api_library.md @@ -0,0 +1,51 @@ + + +# Use Spark Operator API Library in Your Java Application + +In addition to direct YAML, you may also leverage our Java API library to create and manage Spark +workloads. + +## Choosing the Right Dependency + +We publish two variants of this library: + +```groovy +implementation("org.apache.spark:spark-operator-api") +``` + +This library keeps your project free from a direct dependency on Fabric8 Kubernetes Client. It's +recommended if you only interact with our API at the YAML/CRD level (e.g. generating manifests, +using our higher-level abstractions). Use this if you don’t need direct access to Fabric8 model +objects like PodTemplateSpec, Deployment, etc. + +If your application requires access on fabric8 models, it's recommended to use + +```groovy +implementation("org.apache.spark:spark-operator-api-fabric8") +``` + +This library publishes the same core library, with Fabric8 declared as an API dependency. This +means Fabric8 model classes (Pod, PodTemplateSpec, Deployment, etc.) are available directly on +your classpath when you depend on our library. It's recommended if you want to compose or extend +Spark Operator API using Fabric8 types (for example, embedding a Fabric8 PodTemplateSpec into +Spark Operator CRD). + +Do not depend on both `spark-operator-api` and `spark-operator-api-fabric8` at the same time. +Choose one based on your use case. diff --git a/settings.gradle b/settings.gradle index bf5289fb..a123d1ab 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ */ rootProject.name = 'apache-spark-kubernetes-operator' include 'spark-operator-api' +include 'spark-operator-api-fabric8' include 'spark-submission-worker' include 'spark-operator' include "build-tools-docs-utils" diff --git a/spark-operator-api-fabric8/build.gradle b/spark-operator-api-fabric8/build.gradle new file mode 100644 index 00000000..a2e1bcff --- /dev/null +++ b/spark-operator-api-fabric8/build.gradle @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id 'java-library' +} + +dependencies { + api project(":spark-operator-api") + // fabric8 + api(libs.kubernetes.client) { + exclude group: 'com.squareup.okhttp3' + } +} diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle index f9323f27..b515c8ee 100644 --- a/spark-operator-api/build.gradle +++ b/spark-operator-api/build.gradle @@ -22,7 +22,7 @@ plugins { dependencies { // fabric8 - api(libs.kubernetes.client) { + implementation(libs.kubernetes.client) { exclude group: 'com.squareup.okhttp3' } compileOnly(libs.crd.generator.apt) From 62d2e2f70470e369bf438664ce7cf780feb4b59b Mon Sep 17 00:00:00 2001 From: Zhou JIANG Date: Wed, 24 Sep 2025 12:20:25 -0700 Subject: [PATCH 3/3] Fix style --- docs/spark_operator_api_library.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spark_operator_api_library.md b/docs/spark_operator_api_library.md index b6ee2612..e5782912 100644 --- a/docs/spark_operator_api_library.md +++ b/docs/spark_operator_api_library.md @@ -47,5 +47,5 @@ your classpath when you depend on our library. It's recommended if you want to c Spark Operator API using Fabric8 types (for example, embedding a Fabric8 PodTemplateSpec into Spark Operator CRD). -Do not depend on both `spark-operator-api` and `spark-operator-api-fabric8` at the same time. +Do not depend on both `spark-operator-api` and `spark-operator-api-fabric8` at the same time. Choose one based on your use case.