Add field to CRD

sbernauer · sbernauer · commit 48a0a837744c · 2025-03-04T10:48:41.000+01:00
diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml
@@ -256,6 +256,32 @@ spec:
                       default: {}
                       description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.'
                       type: object
+                    jvmArgumentOverrides:
+                      default:
+                        add: []
+                        remove: []
+                        removeRegex: []
+                      description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage.
+                      properties:
+                        add:
+                          default: []
+                          description: JVM arguments to be added
+                          items:
+                            type: string
+                          type: array
+                        remove:
+                          default: []
+                          description: JVM arguments to be removed by exact match
+                          items:
+                            type: string
+                          type: array
+                        removeRegex:
+                          default: []
+                          description: JVM arguments matching any of this regexes will be removed
+                          items:
+                            type: string
+                          type: array
+                      type: object
                     podOverrides:
                       default: {}
                       description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information.
@@ -530,6 +556,32 @@ spec:
                       default: {}
                       description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.'
                       type: object
+                    jvmArgumentOverrides:
+                      default:
+                        add: []
+                        remove: []
+                        removeRegex: []
+                      description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage.
+                      properties:
+                        add:
+                          default: []
+                          description: JVM arguments to be added
+                          items:
+                            type: string
+                          type: array
+                        remove:
+                          default: []
+                          description: JVM arguments to be removed by exact match
+                          items:
+                            type: string
+                          type: array
+                        removeRegex:
+                          default: []
+                          description: JVM arguments matching any of this regexes will be removed
+                          items:
+                            type: string
+                          type: array
+                      type: object
                     podOverrides:
                       default: {}
                       description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information.
@@ -620,6 +672,32 @@ spec:
                       default: {}
                       description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.'
                       type: object
+                    jvmArgumentOverrides:
+                      default:
+                        add: []
+                        remove: []
+                        removeRegex: []
+                      description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage.
+                      properties:
+                        add:
+                          default: []
+                          description: JVM arguments to be added
+                          items:
+                            type: string
+                          type: array
+                        remove:
+                          default: []
+                          description: JVM arguments to be removed by exact match
+                          items:
+                            type: string
+                          type: array
+                        removeRegex:
+                          default: []
+                          description: JVM arguments matching any of this regexes will be removed
+                          items:
+                            type: string
+                          type: array
+                      type: object
                     podOverrides:
                       default: {}
                       description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information.
@@ -1416,6 +1494,32 @@ spec:
                       default: {}
                       description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.'
                       type: object
+                    jvmArgumentOverrides:
+                      default:
+                        add: []
+                        remove: []
+                        removeRegex: []
+                      description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage.
+                      properties:
+                        add:
+                          default: []
+                          description: JVM arguments to be added
+                          items:
+                            type: string
+                          type: array
+                        remove:
+                          default: []
+                          description: JVM arguments to be removed by exact match
+                          items:
+                            type: string
+                          type: array
+                        removeRegex:
+                          default: []
+                          description: JVM arguments matching any of this regexes will be removed
+                          items:
+                            type: string
+                          type: array
+                      type: object
                     podOverrides:
                       default: {}
                       description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information.
@@ -1636,6 +1740,32 @@ spec:
                             default: {}
                             description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.'
                             type: object
+                          jvmArgumentOverrides:
+                            default:
+                              add: []
+                              remove: []
+                              removeRegex: []
+                            description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage.
+                            properties:
+                              add:
+                                default: []
+                                description: JVM arguments to be added
+                                items:
+                                  type: string
+                                type: array
+                              remove:
+                                default: []
+                                description: JVM arguments to be removed by exact match
+                                items:
+                                  type: string
+                                type: array
+                              removeRegex:
+                                default: []
+                                description: JVM arguments matching any of this regexes will be removed
+                                items:
+                                  type: string
+                                type: array
+                            type: object
                           podOverrides:
                             default: {}
                             description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information.
diff --git a/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc b/docs/modules/spark-k8s/pages/usage-guide/overrides.adoc
@@ -148,3 +148,10 @@ spec:
 
 The Spark operator also supports Pod overrides, allowing you to override any property that you can set on a Kubernetes Pod.
 Read the xref:concepts:overrides.adoc#pod-overrides[Pod overrides documentation] to learn more about this feature.
+
+== JVM argument overrides
+
+Stackable operators automatically determine the set of needed JVM arguments, such as memory settings or trust- and keystores.
+Using JVM argument overrides you can configure the JVM arguments xref:concepts:overrides.adoc#jvm-argument-overrides[according to the concepts page].
+
+TODO more docs
diff --git a/docs/modules/spark-k8s/partials/nav.adoc b/docs/modules/spark-k8s/partials/nav.adoc
@@ -10,7 +10,7 @@
 ** xref:spark-k8s:usage-guide/logging.adoc[]
 ** xref:spark-k8s:usage-guide/history-server.adoc[]
 ** xref:spark-k8s:usage-guide/examples.adoc[]
-** xref:spark-k8s:usage-guide/configuration-environment-overrides.adoc[]
+** xref:spark-k8s:usage-guide/overrides.adoc[]
 ** xref:spark-k8s:usage-guide/operations/index.adoc[]
 *** xref:spark-k8s:usage-guide/operations/applications.adoc[]
 *** xref:spark-k8s:usage-guide/operations/pod-placement.adoc[]
diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs
@@ -24,7 +24,7 @@ use stackable_operator::{
         ValidatedRoleConfigByPropertyKind,
     },
     product_logging::{self, spec::Logging},
-    role_utils::{GenericProductSpecificCommonConfig, Role, RoleGroup, RoleGroupRef},
+    role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroup, RoleGroupRef},
     schemars::{self, JsonSchema},
     time::Duration,
 };
@@ -51,6 +51,8 @@ pub enum Error {
 
 #[versioned(version(name = "v1alpha1"))]
 pub mod versioned {
+    use stackable_operator::role_utils::{GenericRoleConfig, JavaCommonConfig};
+
     /// A Spark cluster history server component. This resource is managed by the Stackable operator
     /// for Apache Spark. Find more information on how to use it in the
     /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/history-server).
@@ -86,7 +88,7 @@ pub mod versioned {
         pub spark_conf: BTreeMap<String, String>,
 
         /// A history server node role definition.
-        pub nodes: Role<HistoryConfigFragment>,
+        pub nodes: Role<HistoryConfigFragment, GenericRoleConfig, JavaCommonConfig>,
     }
 
     #[derive(Clone, Deserialize, Debug, Default, Eq, JsonSchema, PartialEq, Serialize)]
@@ -133,15 +135,15 @@ impl CurrentlySupportedListenerClasses {
 
 impl v1alpha1::SparkHistoryServer {
     /// Returns a reference to the role. Raises an error if the role is not defined.
-    pub fn role(&self) -> &Role<HistoryConfigFragment> {
+    pub fn role(&self) -> &Role<HistoryConfigFragment, GenericRoleConfig, JavaCommonConfig> {
         &self.spec.nodes
     }
 
     /// Returns a reference to the role group. Raises an error if the role or role group are not defined.
     pub fn rolegroup(
         &self,
         rolegroup_ref: &RoleGroupRef<Self>,
-    ) -> Result<RoleGroup<HistoryConfigFragment, GenericProductSpecificCommonConfig>, Error> {
+    ) -> Result<RoleGroup<HistoryConfigFragment, JavaCommonConfig>, Error> {
         self.spec
             .nodes
             .role_groups
@@ -207,7 +209,10 @@ impl v1alpha1::SparkHistoryServer {
     ) -> Result<ValidatedRoleConfigByPropertyKind, Error> {
         let roles_to_validate: HashMap<
             String,
-            (Vec<PropertyNameKind>, Role<HistoryConfigFragment>),
+            (
+                Vec<PropertyNameKind>,
+                Role<HistoryConfigFragment, GenericRoleConfig, JavaCommonConfig>,
+            ),
         > = vec![(
             HISTORY_ROLE_NAME.to_string(),
             (
diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs
@@ -37,9 +37,7 @@ use stackable_operator::{
         ValidatedRoleConfigByPropertyKind,
     },
     product_logging,
-    role_utils::{
-        CommonConfiguration, GenericProductSpecificCommonConfig, GenericRoleConfig, Role, RoleGroup,
-    },
+    role_utils::{CommonConfiguration, GenericRoleConfig, JavaCommonConfig, Role, RoleGroup},
     schemars::{self, JsonSchema},
     time::Duration,
     utils::crds::raw_object_list_schema,
@@ -174,19 +172,19 @@ pub struct SparkApplicationSpec {
     /// The reason this property uses its own type (SubmitConfigFragment) is because logging is not
     /// supported for spark-submit processes.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub job: Option<CommonConfiguration<SubmitConfigFragment, GenericProductSpecificCommonConfig>>,
+    pub job: Option<CommonConfiguration<SubmitConfigFragment, JavaCommonConfig>>,
 
     /// The driver role specifies the configuration that, together with the driver pod template, is used by
     /// Spark to create driver pods.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub driver: Option<CommonConfiguration<RoleConfigFragment, GenericProductSpecificCommonConfig>>,
+    pub driver: Option<CommonConfiguration<RoleConfigFragment, JavaCommonConfig>>,
 
     /// The executor role specifies the configuration that, together with the driver pod template, is used by
     /// Spark to create the executor pods.
     /// This is RoleGroup instead of plain CommonConfiguration because it needs to allows for the number of replicas.
     /// to be specified.
     #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub executor: Option<RoleGroup<RoleConfigFragment, GenericProductSpecificCommonConfig>>,
+    pub executor: Option<RoleGroup<RoleConfigFragment, JavaCommonConfig>>,
 
     /// A map of key/value strings that will be passed directly to spark-submit.
     #[serde(default)]
@@ -829,7 +827,7 @@ impl v1alpha1::SparkApplication {
             }
         };
 
-        let executor_conf: RoleGroup<RoleConfigFragment, GenericProductSpecificCommonConfig> =
+        let executor_conf: RoleGroup<RoleConfigFragment, JavaCommonConfig> =
             if self.spec.executor.is_some() {
                 self.spec.executor.as_ref().unwrap().clone()
             } else {
