chore: Version SparkApplication

Author: Techassi

Cargo.lock

@@ -10,19 +10,21 @@ edition = "2021"
 repository = "https://github.com/stackabletech/spark-k8s-operator"
 
 [workspace.dependencies]
+stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" }
+product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
+
 anyhow = "1.0"
 built = { version = "0.7", features = ["chrono", "git2"] }
 clap = "4.5"
 const_format = "0.2"
 futures = { version = "0.3", features = ["compat"] }
-product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
 rstest = "0.24"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.39", features = ["full"] }
 tracing = "0.1"

Cargo.toml

@@ -12,7 +12,7 @@ spec:
     kind: SparkApplication
     plural: sparkapplications
     shortNames:
-      - sc
+      - sparkapp
     singular: sparkapplication
   scope: Namespaced
   versions:

deploy/helm/spark-k8s-operator/crds/crds.yaml

@@ -9,15 +9,17 @@ repository.workspace = true
 publish = false
 
 [dependencies]
+stackable-versioned.workspace = true
+stackable-operator.workspace = true
+product-config.workspace = true
+
 anyhow.workspace = true
 const_format.workspace = true
-product-config.workspace = true
 semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
 snafu.workspace = true
-stackable-operator.workspace = true
 strum.workspace = true
 tracing.workspace = true
 tracing-futures.workspace = true

rust/operator-binary/Cargo.toml

@@ -44,6 +44,7 @@ use stackable_operator::{
     time::Duration,
     utils::crds::raw_object_list_schema,
 };
+use stackable_versioned::versioned;
 
 use crate::crd::roles::{
     RoleConfig, RoleConfigFragment, SparkApplicationRole, SparkContainer, SparkMode, SubmitConfig,
@@ -126,20 +127,21 @@ pub struct SparkApplicationStatus {
 ///
 /// The SparkApplication CRD looks a little different than the CRDs of the other products on the
 /// Stackable Data Platform.
-#[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)]
-#[kube(
-    group = "spark.stackable.tech",
-    version = "v1alpha1",
-    kind = "SparkApplication",
-    shortname = "sc",
-    status = "SparkApplicationStatus",
-    namespaced,
-    crates(
-        kube_core = "stackable_operator::kube::core",
-        k8s_openapi = "stackable_operator::k8s_openapi",
-        schemars = "stackable_operator::schemars"
+#[versioned(
+    version(name = "v1alpha1"),
+    k8s(
+        group = "spark.stackable.tech",
+        shortname = "sparkapp",
+        status = "SparkApplicationStatus",
+        namespaced,
+        crates(
+            kube_core = "stackable_operator::kube::core",
+            k8s_openapi = "stackable_operator::k8s_openapi",
+            schemars = "stackable_operator::schemars"
+        )
     )
 )]
+#[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SparkApplicationSpec {
     /// Mode: cluster or client. Currently only cluster is supported.
@@ -241,7 +243,7 @@ pub struct JobDependencies {
     pub exclude_packages: Vec<String>,
 }
 
-impl SparkApplication {
+impl v1alpha1::SparkApplication {
     /// Returns if this [`SparkApplication`] has already created a Kubernetes Job doing the actual `spark-submit`.
     ///
     /// This is needed because Kubernetes will remove the succeeded Job after some time. When the spark-k8s-operator is
@@ -509,7 +511,7 @@ impl SparkApplication {
         &'a self,
         app_version: &'a str,
         role: &'a str,
-    ) -> ObjectLabels<SparkApplication> {
+    ) -> ObjectLabels<v1alpha1::SparkApplication> {
         ObjectLabels {
             owner: self,
             app_name: APP_NAME,
@@ -1104,7 +1106,7 @@ mod tests {
 
     #[test]
     fn test_default_resource_limits() {
-        let spark_application = serde_yaml::from_str::<SparkApplication>(indoc! {"
+        let spark_application = serde_yaml::from_str::<v1alpha1::SparkApplication>(indoc! {"
             ---
             apiVersion: spark.stackable.tech/v1alpha1
             kind: SparkApplication
@@ -1133,7 +1135,7 @@ mod tests {
 
     #[test]
     fn test_merged_resource_limits() {
-        let spark_application = serde_yaml::from_str::<SparkApplication>(indoc! {r#"
+        let spark_application = serde_yaml::from_str::<v1alpha1::SparkApplication>(indoc! {r#"
             ---
             apiVersion: spark.stackable.tech/v1alpha1
             kind: SparkApplication
@@ -1310,7 +1312,7 @@ mod tests {
 
     #[test]
     fn test_validated_config() {
-        let spark_application = serde_yaml::from_str::<SparkApplication>(indoc! {r#"
+        let spark_application = serde_yaml::from_str::<v1alpha1::SparkApplication>(indoc! {r#"
             ---
             apiVersion: spark.stackable.tech/v1alpha1
             kind: SparkApplication
@@ -1375,7 +1377,7 @@ mod tests {
 
     #[test]
     fn test_job_volume_mounts() {
-        let spark_application = serde_yaml::from_str::<SparkApplication>(indoc! {r#"
+        let spark_application = serde_yaml::from_str::<v1alpha1::SparkApplication>(indoc! {r#"
             ---
             apiVersion: spark.stackable.tech/v1alpha1
             kind: SparkApplication

rust/operator-binary/src/crd/mod.rs

@@ -38,7 +38,7 @@ use stackable_operator::{
 };
 use strum::{Display, EnumIter};
 
-use crate::crd::{ResolvedLogDir, SparkApplication};
+use crate::crd::{v1alpha1, ResolvedLogDir};
 
 #[derive(Clone, Debug, Deserialize, Display, Eq, PartialEq, Serialize, JsonSchema)]
 #[strum(serialize_all = "kebab-case")]
@@ -155,7 +155,7 @@ impl RoleConfig {
     }
     pub fn volume_mounts(
         &self,
-        spark_application: &SparkApplication,
+        spark_application: &v1alpha1::SparkApplication,
         s3conn: &Option<S3ConnectionSpec>,
         logdir: &Option<ResolvedLogDir>,
     ) -> Vec<VolumeMount> {
@@ -165,7 +165,7 @@ impl RoleConfig {
 }
 
 impl Configuration for RoleConfigFragment {
-    type Configurable = SparkApplication;
+    type Configurable = v1alpha1::SparkApplication;
 
     fn compute_env(
         &self,
@@ -246,7 +246,7 @@ impl SubmitConfig {
 }
 
 impl Configuration for SubmitConfigFragment {
-    type Configurable = SparkApplication;
+    type Configurable = v1alpha1::SparkApplication;
 
     fn compute_env(
         &self,

rust/operator-binary/src/crd/roles.rs

@@ -18,7 +18,8 @@ use stackable_operator::{
         },
     },
     logging::controller::report_controller_reconciled,
-    CustomResourceExt,
+    shared::yaml::SerializeOptions,
+    YamlSchema,
 };
 use tracing::info_span;
 use tracing_futures::Instrument;
@@ -29,7 +30,7 @@ use crate::crd::{
         SPARK_CONTROLLER_NAME, SPARK_FULL_CONTROLLER_NAME,
     },
     history::SparkHistoryServer,
-    SparkApplication,
+    v1alpha1, SparkApplication,
 };
 
 mod crd;
@@ -63,8 +64,9 @@ async fn main() -> anyhow::Result<()> {
     let opts = Opts::parse();
     match opts.cmd {
         Command::Crd => {
-            SparkApplication::print_yaml_schema(built_info::PKG_VERSION)?;
-            SparkHistoryServer::print_yaml_schema(built_info::PKG_VERSION)?;
+            SparkApplication::merged_crd(SparkApplication::V1Alpha1)?
+                .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?;
+            // SparkHistoryServer::print_yaml_schema(built_info::PKG_VERSION)?;
         }
         Command::Run(ProductOperatorRun {
             product_config,
@@ -104,7 +106,7 @@ async fn main() -> anyhow::Result<()> {
                 },
             ));
             let app_controller = Controller::new(
-                watch_namespace.get_api::<DeserializeGuard<SparkApplication>>(&client),
+                watch_namespace.get_api::<DeserializeGuard<v1alpha1::SparkApplication>>(&client),
                 watcher::Config::default(),
             )
             .owns(

rust/operator-binary/src/main.rs

@@ -13,7 +13,7 @@ use stackable_operator::{
 };
 use strum::{EnumDiscriminants, IntoStaticStr};
 
-use crate::crd::{constants::POD_DRIVER_CONTROLLER_NAME, SparkApplication, SparkApplicationStatus};
+use crate::crd::{constants::POD_DRIVER_CONTROLLER_NAME, v1alpha1, SparkApplicationStatus};
 
 const LABEL_NAME_INSTANCE: &str = "app.kubernetes.io/instance";
 
@@ -84,7 +84,7 @@ pub async fn reconcile(pod: Arc<DeserializeGuard<Pod>>, client: Arc<Client>) ->
     )?;
 
     let app = client
-        .get::<SparkApplication>(
+        .get::<v1alpha1::SparkApplication>(
             app_name.as_ref(),
             pod.metadata
                 .namespace

rust/operator-binary/src/pod_driver_controller.rs

@@ -59,7 +59,7 @@ use crate::{
         constants::*,
         logdir::ResolvedLogDir,
         roles::{RoleConfig, SparkApplicationRole, SparkContainer, SubmitConfig},
-        tlscerts, to_spark_env_sh_string, SparkApplication, SparkApplicationStatus,
+        tlscerts, to_spark_env_sh_string, v1alpha1, SparkApplicationStatus,
     },
     product_logging::{self, resolve_vector_aggregator_address},
     Ctx,
@@ -206,7 +206,7 @@ impl ReconcilerError for Error {
 }
 
 pub async fn reconcile(
-    spark_application: Arc<DeserializeGuard<SparkApplication>>,
+    spark_application: Arc<DeserializeGuard<v1alpha1::SparkApplication>>,
     ctx: Arc<Ctx>,
 ) -> Result<Action> {
     tracing::info!("Starting reconcile");
@@ -430,7 +430,7 @@ pub async fn reconcile(
 }
 
 fn init_containers(
-    spark_application: &SparkApplication,
+    spark_application: &v1alpha1::SparkApplication,
     logging: &Logging<SparkContainer>,
     s3conn: &Option<S3ConnectionSpec>,
     logdir: &Option<ResolvedLogDir>,
@@ -585,7 +585,7 @@ fn init_containers(
 
 #[allow(clippy::too_many_arguments)]
 fn pod_template(
-    spark_application: &SparkApplication,
+    spark_application: &v1alpha1::SparkApplication,
     role: SparkApplicationRole,
     config: &RoleConfig,
     volumes: &[Volume],
@@ -678,7 +678,7 @@ fn pod_template(
 
 #[allow(clippy::too_many_arguments)]
 fn pod_template_config_map(
-    spark_application: &SparkApplication,
+    spark_application: &v1alpha1::SparkApplication,
     role: SparkApplicationRole,
     merged_config: &RoleConfig,
     product_config: Option<&HashMap<PropertyNameKind, BTreeMap<String, String>>>,
@@ -797,7 +797,7 @@ fn pod_template_config_map(
 }
 
 fn submit_job_config_map(
-    spark_application: &SparkApplication,
+    spark_application: &v1alpha1::SparkApplication,
     product_config: Option<&HashMap<PropertyNameKind, BTreeMap<String, String>>>,
     spark_image: &ResolvedProductImage,
 ) -> Result<ConfigMap> {
@@ -856,7 +856,7 @@ fn submit_job_config_map(
 
 #[allow(clippy::too_many_arguments)]
 fn spark_job(
-    spark_application: &SparkApplication,
+    spark_application: &v1alpha1::SparkApplication,
     spark_image: &ResolvedProductImage,
     serviceaccount: &ServiceAccount,
     env: &[EnvVar],
@@ -974,7 +974,7 @@ fn spark_job(
 /// Both objects have an owner reference to the SparkApplication, as well as the same name as the app.
 /// They are deleted when the job is deleted.
 fn build_spark_role_serviceaccount(
-    spark_app: &SparkApplication,
+    spark_app: &v1alpha1::SparkApplication,
     spark_image: &ResolvedProductImage,
 ) -> Result<(ServiceAccount, RoleBinding)> {
     let sa_name = spark_app.metadata.name.as_ref().unwrap().to_string();
@@ -1029,7 +1029,7 @@ fn security_context() -> PodSecurityContext {
 }
 
 pub fn error_policy(
-    _obj: Arc<DeserializeGuard<SparkApplication>>,
+    _obj: Arc<DeserializeGuard<v1alpha1::SparkApplication>>,
     error: &Error,
     _ctx: Arc<Ctx>,
 ) -> Action {