fix: Spark-connect test timing issues (#597)

* fix: Spark-connect test timing issues

* Update tests/templates/kuttl/spark-connect/20-assert.yaml

Co-authored-by: Sebastian Bernauer <[email protected]>

* Update tests/templates/kuttl/spark-connect/11-assert.yaml

I left out the sparkconnectservers check as we check the statefulset, and that waits for the driver. We can keep this check in, but, just for my understanding, do we *need* both?

Co-authored-by: Sebastian Bernauer <[email protected]>

* added comment about checking executors as well as driver

---------

Co-authored-by: Sebastian Bernauer <[email protected]>

Author: adwk67

rust/operator-binary/src/connect/controller.rs

@@ -315,6 +315,12 @@ pub async fn reconcile(
     let cluster_operation_cond_builder =
         ClusterOperationsConditionBuilder::new(&scs.spec.cluster_operation);
 
+    // TODO: This StatefulSet only contains the driver. We should probably also
+    // consider the state of the executors to determine if the
+    // SparkConnectServer is ready. This depends on the availability and
+    // resilience properties of Spark and could e.g. be "driver and more than
+    // 75% of the executors ready". Special care needs to be taken about
+    // auto-scaling executors in this case (if/once supported).
     let status = SparkConnectServerStatus {
         conditions: compute_conditions(scs, &[&ss_cond_builder, &cluster_operation_cond_builder]),
     };

tests/templates/kuttl/spark-connect/10-assert.yaml

@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 300
+timeout: 600
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -23,20 +23,3 @@ metadata:
   name: spark-connect-server-headless
 spec:
   type: ClusterIP
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 300
-commands:
-  # Test that spark connect executors are running.
-  # Sleep to prevent the following spark connect app from failing
-  # while the spark-connect server is busy setting up the executors.
-  - script: |
-      # wait for the spark-connect CR to become available
-      kubectl wait --for=condition=Available sparkconnectservers/spark-connect --namespace "$NAMESPACE" --timeout=3m
-
-      EXECUTOR_COUNT=$(kubectl get pods -n "$NAMESPACE" --selector 'spark-app-name=spark-connect-server' --field-selector='status.phase=Running' -o NAME|wc -l)
-      test 1 -eq "$EXECUTOR_COUNT"
-
-      # wait a little longer to increase the chance apps being able to connect
-      sleep 5

tests/templates/kuttl/spark-connect/11-assert.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  # Test that spark connect executors are running.
+  # Sleep to prevent the following spark connect app from failing
+  # while the spark-connect server is busy setting up the executors.
+  - script: |
+      # wait for the spark-connect CR to become available
+      kubectl wait --for=condition=Available sparkconnectservers/spark-connect --namespace "$NAMESPACE" --timeout=3m
+
+      # FIXME: As the status currently does not respect the executors state, we wait for them to be ready ourselves
+      # (see TODO comment in code):
+      kubectl wait --for=condition=Ready pod -l spark-app-name=spark-connect-server -n "$NAMESPACE" --timeout=10m
+
+      # wait a little longer to increase the chance apps being able to connect
+      sleep 10

tests/templates/kuttl/spark-connect/20-assert.yaml

@@ -1,9 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-metadata:
-  name: simple-connect-app
-timeout: 300
+timeout: 600
 ---
 apiVersion: batch/v1
 kind: Job

tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2

@@ -54,7 +54,7 @@ spec:
   template:
     spec:
       restartPolicy: OnFailure
-      activeDeadlineSeconds: 100
+      activeDeadlineSeconds: 600
       containers:
         - name: simple-connect-app
 {% if test_scenario['values']['spark-connect-client'].find(",") > 0 %}