feat: run containerdebug in the background

Author: razvan

rust/operator-binary/src/history/history_controller.rs

@@ -527,7 +527,13 @@ fn build_stateful_set(
         .context(InvalidContainerNameSnafu)?
         .image_from_product_image(resolved_product_image)
         .resources(merged_config.resources.clone().into())
-        .command(vec!["/bin/bash".to_string()])
+        .command(vec![
+            "/bin/bash".to_string(),
+            "-x".to_string(),
+            "-euo".to_string(),
+            "pipefail".to_string(),
+            "-c".to_string(),
+        ])
         .args(command_args(log_dir))
         .add_container_port("http", 18080)
         .add_container_port("metrics", METRICS_PORT.into())
@@ -751,10 +757,10 @@ fn command_args(logdir: &ResolvedLogDir) -> Vec<String> {
     }
 
     command.extend(vec![
+        format!("CONTAINERDEBUG_LOG_DIRECTORY={VOLUME_MOUNT_PATH_LOG}/containerdebug containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &"),
         format!("/stackable/spark/sbin/start-history-server.sh --properties-file {VOLUME_MOUNT_PATH_CONFIG}/{SPARK_DEFAULTS_FILE_NAME}"),
     ]);
-
-    vec![String::from("-c"), command.join(" && ")]
+    vec![command.join("\n")]
 }
 
 fn labels<'a, T>(

rust/operator-binary/src/spark_k8s_controller.rs

@@ -594,6 +594,11 @@ fn pod_template(
         .resources(config.resources.clone().into())
         .image_from_product_image(spark_image);
 
+    cb.add_env_var(
+        "_STACKABLE_PRE_HOOK",
+        format!("CONTAINERDEBUG_LOG_DIRECTORY={VOLUME_MOUNT_PATH_LOG}/containerdebug containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &"),
+    );
+
     if config.logging.enable_vector_agent {
         cb.add_env_var(
             "_STACKABLE_POST_HOOK",

tests/templates/kuttl/smoke/06-assert.yaml

@@ -9,3 +9,10 @@ metadata:
   name: spark-history-node-default
 status:
   readyReplicas: 1
+---
+# This test checks if the containerdebug-state.json file is present and valid
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  - script: kubectl exec -n $NAMESPACE --container airflow spark-history-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status

tests/templates/kuttl/smoke/10-assert.yaml

@@ -9,3 +9,10 @@ metadata:
   name: spark-pi-s3-1
 status:
   phase: Succeeded
+---
+# This test checks if the containerdebug-state.json file is present and valid
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  - script: kubectl exec -n $NAMESPACE --container airflow spark-history-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status