fix(test): make spark connect test more robust (#587)

* fix(test): make spark connect test more robust

* review feedback

Author: razvan

tests/templates/kuttl/spark-connect/10-assert.yaml

@@ -18,6 +18,9 @@ commands:
   # Sleep to prevent the following spark connect app from failing
   # while the spark-connect server is busy setting up the executors.
   - script: |
-      sleep 10
-      EXECUTOR_COUNT=$(kubectl get pods -n $NAMESPACE --selector 'spark-app-name=spark-connect-server' --field-selector='status.phase=Running' -o NAME|wc -l)
+      # wait for the spark-connect CR to become available
+      # this is to prevent the spark apps created in the following test steps to fail intermitently
+      kubectl wait --for=condition=Available sparkconnectservers/spark-connect --namespace "$NAMESPACE" --timeout=3m
+
+      EXECUTOR_COUNT=$(kubectl get pods -n "$NAMESPACE" --selector 'spark-app-name=spark-connect-server' --field-selector='status.phase=Running' -o NAME|wc -l)
       test 1 -eq "$EXECUTOR_COUNT"