feat: run containerdebug in the background

rust/crd/src/lib.rs

@@ -397,12 +397,6 @@ impl SparkApplication {
                 mount_path: VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES.into(),
                 ..VolumeMount::default()
             },
-            // This is used at least by the containerdebug process
-            VolumeMount {
-                name: VOLUME_MOUNT_NAME_LOG.into(),
-                mount_path: VOLUME_MOUNT_PATH_LOG.into(),
-                ..VolumeMount::default()
-            },
         ];
 
         tmpl_mounts = self.add_common_volume_mounts(tmpl_mounts, s3conn, logdir, false);
@@ -472,14 +466,16 @@ impl SparkApplication {
                 mount_path: VOLUME_MOUNT_PATH_LOG_CONFIG.into(),
                 ..VolumeMount::default()
             });
-
-            mounts.push(VolumeMount {
-                name: VOLUME_MOUNT_NAME_LOG.into(),
-                mount_path: VOLUME_MOUNT_PATH_LOG.into(),
-                ..VolumeMount::default()
-            });
         }
 
+        // This is used at least by the containerdebug process.
+        // The volume is always there.
+        mounts.push(VolumeMount {
+            name: VOLUME_MOUNT_NAME_LOG.into(),
+            mount_path: VOLUME_MOUNT_PATH_LOG.into(),
+            ..VolumeMount::default()
+        });
+
         if !self.packages().is_empty() {
             mounts.push(VolumeMount {
                 name: VOLUME_MOUNT_NAME_IVY2.into(),
@@ -693,13 +689,25 @@ impl SparkApplication {
     ) -> Vec<EnvVar> {
         let mut e: Vec<EnvVar> = self.spec.env.clone();
 
-        // Needed by the `containerdebug` process running in the background of the `spark-submit`
-        // container to log it's tracing information to.
-        e.push(EnvVar {
-            name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(),
-            value: Some(format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug")),
-            value_from: None,
-        });
+        // These env variables enable the `containerdebug` process in driver and executor pods.
+        // More precisely, this process runs in the background of every `spark` container.
+        // - `CONTAINERDEBUG_LOG_DIRECTORY` - is the location where tracing information from the process
+        // is written. This directory is created by the process it's self.
+        // - `_STACKABLE_PRE_HOOK` - is evaluated by the entrypoint script (run-spark.sh) in the Spark images
+        // before the actual JVM process is started. The result of this evaluation is that the
+        // `containerdebug` process is executed in the background.
+        e.extend(vec![
+            EnvVar {
+                name: "CONTAINERDEBUG_LOG_DIRECTORY".into(),
+                value: Some(format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug")),
+                value_from: None,
+            },
+            EnvVar {
+                name: "_STACKABLE_PRE_HOOK".into(),
+                value: Some(format!( "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &")),
+                value_from: None,
+            },
+        ]);
 
         if self.requirements().is_some() {
             e.push(EnvVar {

rust/operator-binary/src/spark_k8s_controller.rs

@@ -612,31 +612,6 @@ fn pod_template(
         .resources(config.resources.clone().into())
         .image_from_product_image(spark_image);
 
-    // These env variables enable the `containerdebug` process in driver and executor pods.
-    // More precisely, this process runs in the background of every `spark` container.
-    // - `CONTAINERDEBUG_LOG_DIRECTORY` - is the location where tracing information from the process
-    // is written. This directory is created by the process it's self.
-    // - `_STACKABLE_PRE_HOOK` - is evaluated by the entrypoint script (run-spark.sh) in the Spark images
-    // before the actual JVM process is started. The result of this evaluation is that the
-    // `containerdebug` process is executed in the background.
-    cb.add_env_vars(
-        [
-            EnvVar {
-                name: "CONTAINERDEBUG_LOG_DIRECTORY".into(),
-                value: Some(format!("{VOLUME_MOUNT_PATH_LOG}/containerdebug")),
-                value_from: None,
-            },
-            EnvVar {
-                name: "_STACKABLE_PRE_HOOK".into(),
-                value: Some(format!(
-            "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &"
-        )),
-                value_from: None,
-            },
-        ]
-        .into(),
-    );
-
     if config.logging.enable_vector_agent {
         cb.add_env_var(
             "_STACKABLE_POST_HOOK",

tests/templates/kuttl/smoke/40-assert.yaml

@@ -9,10 +9,3 @@ metadata:
   name: spark-history-node-default
 status:
   readyReplicas: 1
----
-# This test checks if the containerdebug-state.json file is present and valid
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 60
-commands:
-  - script: kubectl exec -n $NAMESPACE --container spark-history spark-history-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status

tests/templates/kuttl/smoke/41-assert.yaml

@@ -0,0 +1,7 @@
+---
+# This test checks if the containerdebug-state.json file is present and valid
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 60
+commands:
+  - script: kubectl exec -n $NAMESPACE --container spark-history spark-history-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"'

tests/templates/kuttl/smoke/50-assert.yaml

@@ -12,4 +12,4 @@ kind: TestAssert
 commands:
   - script: |
       SPARK_SUBMIT_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector batch.kubernetes.io/job-name=spark-pi-s3-1 -o jsonpath='{.items[0].metadata.name}')
-      kubectl exec -n $NAMESPACE --container spark-submit $SPARK_SUBMIT_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status
+      kubectl exec -n $NAMESPACE --container spark-submit $SPARK_SUBMIT_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"'