[SPARK-54328] Add configurable startupProbe and enhance liveness/readiness probes in Helm chart

### What changes were proposed in this pull request?

This PR adds a configurable `startupProbe` and enhances the existing `livenessProbe` and `readinessProbe` with additional configurable parameters in the Helm chart for the `spark-kubernetes-operator`.

### Why are the changes needed?

Previously, these values were either using Kubernetes defaults or not configured at all. This change makes them explicitly configurable via Helm values, giving operators more control over pod lifecycle management in different cluster environments (small clusters vs large production clusters).

### Does this PR introduce _any_ user-facing change?

Yes. Users can now configure these probe settings in their `values.yaml`.

### How was this patch tested?

E2E coverage for default value, and local dry-run for value overrides.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #417 from jiangzho/probe_override.

Authored-by: Zhou JIANG <[email protected]>
Signed-off-by: Peter Toth <[email protected]>

Author: jiangzho

build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl

@@ -125,10 +125,13 @@ spark.kubernetes.operator.watchedNamespaces={{ include "spark-operator.workloadN
 Readiness Probe properties overrides
 */}}
 {{- define "spark-operator.readinessProbe.failureThreshold" -}}
-{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.failureThreshold }}
+{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.readinessProbe.failureThreshold }}
 {{- end }}
 {{- define "spark-operator.readinessProbe.periodSeconds" -}}
-{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.periodSeconds }}
+{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.readinessProbe.periodSeconds }}
+{{- end }}
+{{- define "spark-operator.readinessProbe.timeoutSeconds" -}}
+{{- default 1 .Values.operatorDeployment.operatorPod.operatorContainer.probes.readinessProbe.timeoutSeconds }}
 {{- end }}
 
 {{/*
@@ -140,6 +143,28 @@ Liveness Probe properties override
 {{- define "spark-operator.livenessProbe.periodSeconds" -}}
 {{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.periodSeconds }}
 {{- end }}
+{{- define "spark-operator.livenessProbe.failureThreshold" -}}
+{{- default 1 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.failureThreshold }}
+{{- end }}
+{{- define "spark-operator.livenessProbe.timeoutSeconds" -}}
+{{- default 1 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.timeoutSeconds }}
+{{- end }}
+
+{{/*
+Startup Probe properties override
+*/}}
+{{- define "spark-operator.startupProbe.initialDelaySeconds" -}}
+{{- default 0 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.initialDelaySeconds }}
+{{- end }}
+{{- define "spark-operator.startupProbe.failureThreshold" -}}
+{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.failureThreshold }}
+{{- end }}
+{{- define "spark-operator.startupProbe.periodSeconds" -}}
+{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.periodSeconds }}
+{{- end }}
+{{- define "spark-operator.startupProbe.timeoutSeconds" -}}
+{{- default 1 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.timeoutSeconds }}
+{{- end }}
 
 {{/*
 Readiness Probe property overrides

build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml

@@ -116,18 +116,29 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          startupProbe:
+            httpGet:
+              port: probe-port
+              path: /healthz
+            initialDelaySeconds: {{ include "spark-operator.startupProbe.initialDelaySeconds" . }}
+            failureThreshold: {{ include "spark-operator.startupProbe.failureThreshold" . }}
+            periodSeconds: {{ include "spark-operator.startupProbe.periodSeconds" . }}
+            timeoutSeconds: {{ include "spark-operator.startupProbe.timeoutSeconds" . }}
           readinessProbe:
             httpGet:
               port: probe-port
               path: /readyz
             failureThreshold: {{ include "spark-operator.readinessProbe.failureThreshold" . }}
             periodSeconds: {{ include "spark-operator.readinessProbe.periodSeconds" . }}
+            timeoutSeconds: {{ include "spark-operator.readinessProbe.timeoutSeconds" . }}
           livenessProbe:
             httpGet:
               port: probe-port
               path: /healthz
             initialDelaySeconds: {{ include "spark-operator.livenessProbe.initialDelaySeconds" . }}
             periodSeconds: {{ include "spark-operator.livenessProbe.periodSeconds" . }}
+            failureThreshold: {{ include "spark-operator.livenessProbe.failureThreshold" . }}
+            timeoutSeconds: {{ include "spark-operator.livenessProbe.timeoutSeconds" . }}
           {{- with .Values.operatorDeployment.operatorPod.operatorContainer.securityContext }}
           securityContext:
             {{- toYaml . | nindent 12 }}

build-tools/helm/spark-kubernetes-operator/values.yaml

@@ -61,9 +61,14 @@ operatorDeployment:
         livenessProbe:
           periodSeconds: 10
           initialDelaySeconds: 30
+          failureThreshold: 1
+          timeoutSeconds: 1
         startupProbe:
           failureThreshold: 30
           periodSeconds: 10
+        readinessProbe:
+          failureThreshold: 1
+          periodSeconds: 10
       metrics:
         port: 19090
       # By default, operator container is configured to comply restricted standard