successfully create driver and executors

Author: razvan

rust/operator-binary/src/connect/controller.rs

@@ -72,10 +72,7 @@ use crate::{
 #[strum_discriminants(derive(IntoStaticStr))]
 #[allow(clippy::enum_variant_names)]
 pub enum Error {
-    #[snafu(display("missing secret lifetime"))]
-    MissingSecretLifetime,
-
-    #[snafu(display("object has no namespace"))]
+    #[snafu(display("spark connect object has no namespace"))]
     ObjectHasNoNamespace,
 
     #[snafu(display("invalid config map {name}"))]
@@ -271,17 +268,6 @@ pub async fn reconcile(
         .context(ProductConfigValidationSnafu)?
         .iter()
     {
-        let service = build_service(
-            scs,
-            &resolved_product_image.app_version_label,
-            role_name,
-            None,
-        )?;
-        cluster_resources
-            .add(client, service)
-            .await
-            .context(ApplyServiceSnafu)?;
-
         for (rolegroup_name, rolegroup_config) in role_config.iter() {
             let rgr = RoleGroupRef {
                 cluster: ObjectRef::from_obj(scs),
@@ -515,7 +501,12 @@ fn build_stateful_set(
             "pipefail".to_string(),
             "-c".to_string(),
         ])
-        .args(command_args(resolved_product_image))
+        .args(command_args(
+            rolegroupref.object_name().as_ref(),
+            scs.namespace().context(ObjectHasNoNamespaceSnafu)?.as_ref(),
+            resolved_product_image,
+            serviceaccount,
+        ))
         .add_container_port("grpc", CONNECT_GRPC_PORT)
         .add_container_port("http", CONNECT_UI_PORT)
         .add_container_port("metrics", METRICS_PORT.into())
@@ -669,15 +660,41 @@ fn spark_defaults(
     Ok("".to_string())
 }
 
-fn command_args(pi: &ResolvedProductImage) -> Vec<String> {
-    let mut command = vec![];
-
+fn command_args(
+    rgr_name: &str,
+    namespace: &str,
+    pi: &ResolvedProductImage,
+    service_account: &ServiceAccount,
+) -> Vec<String> {
     let spark_version = pi.product_version.clone();
-    command.extend(vec![
-        format!("containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &"),
-        format!("/stackable/spark/sbin/start-connect-server.sh --packages org.apache.spark:spark-connect_2.12:{spark_version} --properties-file {VOLUME_MOUNT_PATH_CONFIG}/{SPARK_DEFAULTS_FILE_NAME}"),
-    ]);
-    vec![command.join("\n")]
+    let spark_image = pi.image.clone();
+    let service_account_name = service_account.name_unchecked();
+
+    let command = [
+        // ---------- start containerdebug
+        format!(
+            "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &"
+        ),
+        // ---------- start spark connect server
+        "/stackable/spark/sbin/start-connect-server.sh".to_string(),
+        "--deploy-mode client".to_string(), // 'cluster' mode not supported
+        format!("--name {rgr_name}"),
+        "--master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
+            .to_string(),
+        // This needs to match the name of the headless service for the executors to be able
+        // to connect back to the driver.
+        format!("--conf spark.driver.host={rgr_name}"),
+        "--conf spark.kubernetes.driver.pod.name=${HOSTNAME}".to_string(),
+        format!("--conf spark.kubernetes.container.image={spark_image}"),
+        format!("--conf spark.kubernetes.namespace={namespace}"),
+        format!(
+            "--conf spark.kubernetes.authenticate.driver.serviceAccountName={service_account_name}"
+        ),
+        format!("--packages org.apache.spark:spark-connect_2.12:{spark_version}"),
+        format!("--properties-file {VOLUME_MOUNT_PATH_CONFIG}/{SPARK_DEFAULTS_FILE_NAME}"),
+    ];
+
+    vec![command.join(" ")]
 }
 
 fn labels<'a, T>(

rust/operator-binary/src/connect/crd.rs

@@ -263,8 +263,13 @@ impl v1alpha1::SparkConnectServer {
                 "SPARK_DAEMON_CLASSPATH".to_string(),
                 "/stackable/spark/extra-jars/*".to_string(),
             ),
-            // TODO: There is no SPARK_CONNECT_OPTS env var.
+            // There is no SPARK_CONNECT_OPTS env var.
             ("SPARK_DAEMON_JAVA_OPTS".to_string(), connect_jvm_args),
+            // TODO: remove when this is done in the image
+            (
+                "JAVA_HOME".to_string(),
+                "/usr/lib/jvm/java-17-openjdk".to_string(),
+            ),
         ]);
 
         envs.extend(role.config.env_overrides.clone());
@@ -362,7 +367,7 @@ impl ConnectConfig {
                     max: Some(Quantity("1".to_owned())),
                 },
                 memory: MemoryLimitsFragment {
-                    limit: Some(Quantity("512Mi".to_owned())),
+                    limit: Some(Quantity("1024Mi".to_owned())),
                     runtime_limits: NoRuntimeLimitsFragment {},
                 },
                 storage: ConnectStorageConfigFragment {},

tests/templates/kuttl/spark-connect/00-secrets.yaml.j2

@@ -19,26 +19,6 @@ stringData:
   accessKey: spark
   secretKey: sparkspark
 ---
-apiVersion: secrets.stackable.tech/v1alpha1
-kind: SecretClass
-metadata:
-  name: history-credentials-class
-spec:
-  backend:
-    k8sSearch:
-      searchNamespace:
-        pod: {}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: history-credentials
-  labels:
-    secrets.stackable.tech/class: history-credentials-class
-stringData:
-  accessKey: spark
-  secretKey: sparkspark
----
 apiVersion: v1
 kind: Secret
 metadata:
@@ -51,28 +31,3 @@ stringData:
     disabled=false
     policies=readwrite,consoleAdmin,diagnostics
     setPolicies=false
-
-{% if test_scenario['values']['s3-use-tls'] == 'true' %}
----
-apiVersion: secrets.stackable.tech/v1alpha1
-kind: SecretClass
-metadata:
-  name: minio-tls-eventlog
-spec:
-  backend:
-    k8sSearch:
-      searchNamespace:
-        pod: {}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: minio-tls-eventlog
-  labels:
-    secrets.stackable.tech/class: minio-tls-eventlog
-# Have a look at the folder certs on how to create this: ensure the common name matches the minio instance!
-data:
-  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQyVENDQXNHZ0F3SUJBZ0lVZEhBWE16UHJaTGx1TURPNlhrT1ozQUdJaGlzd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVGt4TlRJek16UmEKR0E4eU1USXpNRFV5TmpFMU1qTXpORm93ZXpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4S0RBbUJnTlZCQW9NSDFOMFlXTnJZV0pzClpTQlRhV2R1YVc1bklFRjFkR2h2Y21sMGVTQkpibU14RlRBVEJnTlZCQU1NREhOMFlXTnJZV0pzWlM1a1pUQ0MKQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFOR0Jqang2K1lCSm1jOUJaeE5QdnlnRwo2b3VGWFNlajBFWVFISlRyQStZWW5VeGV4eEwyYUU2R2hHTEJWTEYzYkFpUXErbUV0WDlZZGdaRFhBTy9aS1N6Cjl5dXBXY2dkR1B0Uk9qQ0RYdE1SY1pWNFhBRVNsODBRZkVhL041U2ZjTHJBM0JNNWp3YzlrcGR5TXgzOWJkbEEKRExvMlN2d0czTmdOeUpWZkRaS2N5MExFZThnaHFOVFcwSHhKN3FMdHFhb0cycVl2SUVUaDUyd3RzOTUyYlZWMgo3aW9kOUhZWUx2dzFVdzQ1bXM4bXViYldVTHNRMXNORmxRTDAvOU1SakdMREM2RHc5NW9wMzA4MkxXU0NHZ0JMCis4eCtCVTVoUFRGeDN6VFZDVFI1WCtvS0Y4SjlOREZmcjVLK1lUZE5QMG5PRUdOcXcrdDBQcWYyR3N4MWFnOEMKQXdFQUFhTlRNRkV3SFFZRFZSME9CQllFRkFNMFNKaTB3WWpIcWwzVDYwVk0xb251VWhCTk1COEdBMVVkSXdRWQpNQmFBRkFNMFNKaTB3WWpIcWwzVDYwVk0xb251VWhCTk1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJCmh2Y05BUUVMQlFBRGdnRUJBRXVsdzVqeUlpQmFNNHVzSlFaYUNoV1BPZlFqbzBoRGNFL2REWnZNSjVhMEF4WEkKWHFOd284c3B1WlAzSkY4cks3SlpmYmdKcjJzV2FYUFZCR3hENk56WTBpTWhCMTFQNVZOZ3JpczFFeDJCTmZCdQpGSitvUHdjNnBGWjVsbTl3SVpKRk9USXk1djRyOG1tVCtscnFSYzZERG1LWlpCWVB0NVE1a1pXQm54TmlLTGVHCnVaYXhONFVCRnBLdlpqOHdRMi81amxGWXh6R3Z3NjRPVTIvN3N0NGJuU2NWbEhYa0cvWHJsYVg4NmJoREFUcDYKeEtsQWxhMkJEeVhJRWs1R2Rma2lmTG1VRTZZSGozUUJBd0prMkFYQTNMaElGY2NxRkNwclFLblpJcHNuSjRzMwprNXZSWDVibWkyUGhLc2Y3bmR5M0tMaHVUYlBQcnE2UGtuZkpzd3c9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQyakNDQXNLZ0F3SUJBZ0lVRlI3RHEvdlhUSUpXcEE2MEppL0paUTFKRzlVd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVGt4TlRJek16UmEKR0E4eU1USXpNRFV5TmpFMU1qTXpORm93WnpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4RWpBUUJnTlZCQW9NQ1ZOMFlXTnJZV0pzClpURVhNQlVHQTFVRUF3d09aWFpsYm5Sc2IyY3RiV2x1YVc4d2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUIKRHdBd2dnRUtBb0lCQVFESFN6NEJ5UzRORjM0emhleXBDMllsTCt5VEtEbC9WQVBEaHF1dE5TZTh2MkdzdXJZdgpDVDdjZmw2MzFETHc5eHc4NlVDRzVKMVc3a25uZGNlMExEcGhwWmNiUVUzcVNMcisxRG1HdHc5MWdKclFoREg5CjhiZXgxbkN1NExVSllWdldCVGg2TUNhdFVkYmdvb09DNGExdHJmeDJ5VHhDZEZ5cVlHUm1SeU9oNVFxZEF1VlIKWjV0bG9FU1BJRFVJVnhVQk1iZ1BlbUt6L0hhK2xkcDhkVzZTS1ZqWml1TnZxaTI5UkU0bUNyLzR4TXlMZWwwVgp1TllNcHFPcVprWEczbVpqVWhsRWpSSThJMSs1OU13WVBYZVowS1hKK2J6cmw4SWZVNU15NkZHRkkvalovVXBxCnIxcitOYU9zWmY4MXMvK0dyUi95UkJ2bkNaWlBOS0tpVWFpUkFnTUJBQUdqYURCbU1DUUdBMVVkRVFRZE1CdUMKRG1WMlpXNTBiRzluTFcxcGJtbHZnZ2xzYjJOaGJHaHZjM1F3SFFZRFZSME9CQllFRk55aTMrb0ZRNHpKaWdCdQo2VjJEejVWNnB6Z09NQjhHQTFVZEl3UVlNQmFBRkFNMFNKaTB3WWpIcWwzVDYwVk0xb251VWhCTk1BMEdDU3FHClNJYjNEUUVCQ3dVQUE0SUJBUURIeEhJNzZoZUt1cndBRUZQelVoNjdiT2ViM0UxeGtyb0RnR29ZZzIyOWJCVXgKS0JPZWJvUUhuK0JjTEtCeUFXcEphWnBIajAzTldOS0IxcTM4YUx6UXpqbkhlMDZCWU9IQ1kwRFJsREZ1YlhnMgpUQXQyb25vN1ZWNStVTnNGSU5ONnFuWFlDSnV3L2N2WnJVZ3p2MXI5dDJyWnhHVllaVWppeEJRMjRrK0hsdnlGClN6WnBjU3Y1ZlRPMGd6ZUJTaEZkR09FME1HU0x5RGZoT1ZoZGRhanRyN0E1NEVITFFkME5BcXNKQjB3blo4eEgKZWZ4VGhZTktxZEJkaitZd0xGMFk0TWtDaGYva2t4dEhLT012Q2lBS0F3YkEwZ25hT0RDeEd2VEZiSFFFOW5zMgpCQUZqblZKNGh3aTQzeDkxOTFjQnVqOGdZQkdpaUh1NCtGRlNXMXNSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRREhTejRCeVM0TkYzNHoKaGV5cEMyWWxMK3lUS0RsL1ZBUERocXV0TlNlOHYyR3N1cll2Q1Q3Y2ZsNjMxREx3OXh3ODZVQ0c1SjFXN2tubgpkY2UwTERwaHBaY2JRVTNxU0xyKzFEbUd0dzkxZ0pyUWhESDk4YmV4MW5DdTRMVUpZVnZXQlRoNk1DYXRVZGJnCm9vT0M0YTF0cmZ4MnlUeENkRnlxWUdSbVJ5T2g1UXFkQXVWUlo1dGxvRVNQSURVSVZ4VUJNYmdQZW1Lei9IYSsKbGRwOGRXNlNLVmpaaXVOdnFpMjlSRTRtQ3IvNHhNeUxlbDBWdU5ZTXBxT3Faa1hHM21aalVobEVqUkk4STErNQo5TXdZUFhlWjBLWEorYnpybDhJZlU1TXk2RkdGSS9qWi9VcHFyMXIrTmFPc1pmODFzLytHclIveVJCdm5DWlpQCk5LS2lVYWlSQWdNQkFBRUNnZ0VBQVNTUUFHYit2dXZaSVJyaDc1bjVjZW9GZ0haSEJxRTA1SkdVa1ZHa1JzRisKOGhSdnhDTW9uMjl3MC90cE41OG1JLzRITFFMWmVXQzBUeW5LYlhEQnZMSzhOQ2QrcHJMWU1pc0Z3YTFSN3crLwp5NFZ2aXpRUU5HcU5NcjZHZ1QxVC8wMGdQTmlpOXlSWWora1VxWnJwVVg5QmV1Mm1YUi94VWxlTFNkZ2JkdU9nCmpidGJEcTZTNE9GTTVsUEdId09CZXRHVVhVbHd1NENrTTZCQngrYnZPK2UrOXIwZ3BhdFp3QUl4U2djcFR0TkcKVnJqdjdaVmtGK0JwMElZaEUwRCtIc25uemw5cUlZN2hTWEZpQWd2WlJ3bXBndnByMGh2UVl4ZHRqZkpkc0tRYQpRcWo5MnBNYmZJMlZNNjJHbFhFbEcxQXpxZmJDTGsrZHJtc2tXdnpyUVFLQmdRRDhkTm1HQndLL2dpMk9vM1AyCmNQTlZIaDh6M3A3Z2VwM3lmbEZFZ0RkUGhBWk5TUHRkN0NNeU9PUmh0ZDRYdGJWampGRlR2VzB6aVR5ancvL0QKOUgvdWNES1AycjNmcjZJbHNmQ2hBWVd3SlI4eVlGU1V0VUp6TjV3Zmdnb2phUHdUWDZzVE5oMkFNOGtlU3RiegpjSFNRQVd3SjVUUEVoODRKTTNJazlhcXBVUUtCZ1FES0YxMG9tWE5WOEM4S0lZbjI5S0JXWVNNcWZnelcyNyt6CjRVVWVIWGxyZ0JSblVxNHE3UklneDg4alZqSDJrTDZwbFlQMVJ6ZjVBemZOUjNkQnFCYkdHRkRRNEdZcDdNSUwKbmpzZmJkWnN6cHZmNGNBemVXR2RlZ3NOeEpkcDFrNEdhL2ppa0psNTg4MDBSUUd3NVdmdTdvQTlVY29KbFhERwpReTBmbjhjN1FRS0JnUUROQkgvYVRqczNGeHEwS1oxMDEwbGxLRzhpaW5udnh1UWFGK2NiZ0J4cTRNYlZZSjF5Ckt1QnlXcTF0aEdwMlNzU2hzVVFpZzBUNWcwdlJtc3hNY0hTc1NFVm5lYmFVdjJjb3daYXV5ak90VnBnOUt1bkYKVW1aQ2R6aThoYmlDQlRJZzhhSldtYTNmZlM2cEFSSUs1SnBra0NNUzdpWGFSb2RXQllkKzZGck40UUtCZ0NKdQpoS0xNTWtoc0diZWlLejEyUkhNekxhaXB2VXBHT28xTlRsZHBNaXpNcjYxQyt3VzVTcU5uSjdXZmsvZXNoYStqCkY4Nk1pYzdWYTRhQUxLTjFIaHcyUEY2NlBPM2J0dHYvNDViaFFlMWI3bGJnd2J5RGE4eEc1T2M5bllhY3lzSngKVndCVEhyTVdoTm5vLy9iQnYwbUlQVFUxWURUdHp6OFladkhDYVZHQkFvR0FPcGVpaXV1MG1xRXBQR1dXZ0Z5RQplb2FrQ1E5a21MLzM3aTlBaGh4dXI3Q1pvS01kU1B2SHJ2S1Q1UVB4d1UvQ1YweUpBNlpTOWszVzN5bUJlcHNRCjlkMlh1azVDYlp0cVBIWDdoYmxwUExiK1BzTlg0L2JReEJLdTBRR3EzMkJsQU9yMnlFNHBpWFE3YzZRMHQ2MnEKOFlZd1ZaWG1obWdDSTgxMXlkWG9oM1k9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
-{% endif %}

tests/templates/kuttl/spark-connect/03-assert.yaml

@@ -10,13 +10,6 @@ metadata:
 status:
   readyReplicas: 1
 ---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: eventlog-minio
-status:
-  readyReplicas: 1
----
 apiVersion: v1
 kind: Pod
 metadata:

tests/templates/kuttl/spark-connect/03-setup-minio.yaml

@@ -2,13 +2,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: >-
-      helm install eventlog-minio
-      --namespace $NAMESPACE
-      --version 14.6.16
-      -f helm-bitnami-eventlog-minio-values.yaml
-      --repo https://charts.bitnami.com/bitnami minio
-    timeout: 240
   - script: >-
       helm install test-minio
       --namespace $NAMESPACE

tests/templates/kuttl/spark-connect/10-deploy-spark-connect.yaml.j2

@@ -15,7 +15,7 @@ data:
     appender.CONSOLE.filter.threshold.type = ThresholdFilter
     appender.CONSOLE.filter.threshold.level = DEBUG
 
-    rootLogger.level=DEBUG
+    rootLogger.level=INFO
     rootLogger.appenderRefs = CONSOLE
     rootLogger.appenderRef.CONSOLE.ref = CONSOLE
 ---