update

Signed-off-by: bitliu <[email protected]>

Author: Xunzhuo

e2e/profiles/ai-gateway/testcases.go

@@ -72,15 +72,23 @@ func testChatCompletionsRequest(ctx context.Context, client *kubernetes.Clientse
 	for {
 		envoyService, err = getEnvoyServiceName(ctx, opts.Verbose)
 		if err == nil {
-			break
+			// Verify that at least one pod is running for this service
+			podErr := verifyServicePodsRunning(ctx, client, "envoy-gateway-system", envoyService, opts.Verbose)
+			if podErr == nil {
+				break
+			}
+			if opts.Verbose {
+				fmt.Printf("[Test] Envoy service found but pods not ready: %v\n", podErr)
+			}
+			err = fmt.Errorf("service pods not ready: %w", podErr)
 		}
 
 		if time.Since(startTime) >= retryTimeout {
-			return fmt.Errorf("failed to get Envoy service name after %v: %w", retryTimeout, err)
+			return fmt.Errorf("failed to get Envoy service with running pods after %v: %w", retryTimeout, err)
 		}
 
 		if opts.Verbose {
-			fmt.Printf("[Test] Envoy service not found, retrying in %v... (elapsed: %v)\n",
+			fmt.Printf("[Test] Envoy service not ready, retrying in %v... (elapsed: %v)\n",
 				retryInterval, time.Since(startTime).Round(time.Second))
 		}
 
@@ -188,6 +196,66 @@ func getEnvoyServiceName(ctx context.Context, verbose bool) (string, error) {
 	return serviceName, nil
 }
 
+func verifyServicePodsRunning(ctx context.Context, client *kubernetes.Clientset, namespace, serviceName string, verbose bool) error {
+	// Get the service to find its selector
+	svc, err := client.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get service: %w", err)
+	}
+
+	if len(svc.Spec.Selector) == 0 {
+		return fmt.Errorf("service has no selector")
+	}
+
+	// Convert selector map to label selector string
+	var selectorParts []string
+	for key, value := range svc.Spec.Selector {
+		selectorParts = append(selectorParts, fmt.Sprintf("%s=%s", key, value))
+	}
+	labelSelector := strings.Join(selectorParts, ",")
+
+	// List pods matching the service selector
+	pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: labelSelector,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to list pods: %w", err)
+	}
+
+	if len(pods.Items) == 0 {
+		return fmt.Errorf("no pods found for service %s", serviceName)
+	}
+
+	// Check if at least one pod is running
+	runningCount := 0
+	for _, pod := range pods.Items {
+		if pod.Status.Phase == "Running" {
+			// Also check if all containers are ready
+			allReady := true
+			for _, containerStatus := range pod.Status.ContainerStatuses {
+				if !containerStatus.Ready {
+					allReady = false
+					break
+				}
+			}
+			if allReady {
+				runningCount++
+			}
+		}
+	}
+
+	if runningCount == 0 {
+		return fmt.Errorf("no running pods found for service %s (total pods: %d)", serviceName, len(pods.Items))
+	}
+
+	if verbose {
+		fmt.Printf("[Test] Service %s/%s has %d running pod(s) out of %d total\n",
+			namespace, serviceName, runningCount, len(pods.Items))
+	}
+
+	return nil
+}
+
 func startPortForward(ctx context.Context, namespace, service, ports string, verbose bool) error {
 	cmd := exec.CommandContext(ctx, "kubectl", "port-forward",
 		"-n", namespace,