wip

Author: maltesander

rust/operator-binary/src/connect/server.rs

@@ -613,6 +613,7 @@ fn probe() -> Probe {
     }
 }
 
+// TODO: remove this and all rolegroup references
 fn dummy_role_group_ref(
     scs: &v1alpha1::SparkConnectServer,
 ) -> RoleGroupRef<v1alpha1::SparkConnectServer> {
@@ -628,7 +629,9 @@ pub(crate) fn build_listener(
     role_config: &v1alpha1::SparkConnectServerRoleConfig,
     resolved_product_image: &ResolvedProductImage,
 ) -> Result<listener::v1alpha1::Listener, Error> {
+    //let listener_name = server_listener_name(scs);
     let listener_name = dummy_role_group_ref(scs).object_name();
+
     let listener_class = role_config.listener_class.clone();
     let role = SparkConnectRole::Server.to_string();
     let recommended_object_labels =
@@ -656,3 +659,10 @@ pub(crate) fn build_listener(
     )
     .context(BuildListenerSnafu)
 }
+
+fn server_listener_name(scs: &v1alpha1::SparkConnectServer) -> String {
+    format!(
+        "{cluster_name}-server",
+        cluster_name = ObjectRef::from_obj(scs)
+    )
+}

rust/operator-binary/src/crd/history.rs

@@ -83,14 +83,6 @@ pub mod versioned {
     pub struct SparkHistoryServerSpec {
         pub image: ProductImage,
 
-        /// Global Spark history server configuration that applies to all roles.
-        ///
-        /// This was previously used to hold the listener configuration, which has since moved
-        /// to the role configuration.
-        // TODO: remove?
-        #[serde(default)]
-        pub cluster_config: v1alpha1::SparkHistoryServerClusterConfig,
-
         /// Name of the Vector aggregator discovery ConfigMap.
         /// It must contain the key `ADDRESS` with the address of the Vector aggregator.
         #[serde(skip_serializing_if = "Option::is_none")]
@@ -107,10 +99,6 @@ pub mod versioned {
         pub nodes: Role<HistoryConfigFragment, SparkHistoryServerRoleConfig, JavaCommonConfig>,
     }
 
-    #[derive(Clone, Deserialize, Debug, Default, Eq, JsonSchema, PartialEq, Serialize)]
-    #[serde(rename_all = "camelCase")]
-    pub struct SparkHistoryServerClusterConfig {}
-
     // TODO: move generic version to op-rs?
     #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
     #[serde(rename_all = "camelCase")]

tests/templates/kuttl/spark-connect/10-assert.yaml

@@ -10,6 +10,13 @@ metadata:
 status:
   readyReplicas: 1
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-connect-server
+spec:
+  type: NodePort
+---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 300

tests/templates/kuttl/spark-connect/10-deploy-spark-connect.yaml.j2

@@ -62,8 +62,9 @@ spec:
     jvmArgumentOverrides:
       add:
         - -Dmy.custom.jvm.arg=customValue
-    config:
+    roleConfig:
       listenerClass: external-unstable
+    config:  
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
         containers:

tests/templates/kuttl/spark-history-server/06-deploy-history-server.yaml.j2

@@ -44,8 +44,9 @@ spec:
   # For possible properties see: https://spark.apache.org/docs/latest/monitoring.html#spark-history-server-configuration-options
   #sparkConf:
   nodes:
-    config:
+    roleConfig:
       listenerClass: external-unstable
+    config:  
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
         containers: