Skip to content

Commit e554406

Browse files
XunzhuoXunzhuo
committed
update
Signed-off-by: bitliu <[email protected]>
1 parent a326ea4 commit e554406

File tree

5 files changed

+682
-195
lines changed

5 files changed

+682
-195
lines changed

‎.github/workflows/integration-test-ai-gateway.yml‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -64,36 +64,57 @@ jobs:
6464
- name: Run AI Gateway E2E tests
6565
id: e2e-test
6666
run: |
67+
set +e # Don't exit on error, we want to capture the result
6768
make e2e-test PROFILE=ai-gateway
69+
TEST_EXIT_CODE=$?
70+
echo "test_exit_code=${TEST_EXIT_CODE}" >> $GITHUB_OUTPUT
71+
exit ${TEST_EXIT_CODE}
6872
env:
6973
E2E_VERBOSE: "true"
7074
KEEP_CLUSTER: "true"
7175

72-
- name: Show cluster logs on failure
73-
if: failure()
76+
- name: Upload test reports
77+
if: always()
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: test-reports
81+
path: |
82+
test-report.json
83+
test-report.md
84+
retention-days: 30
85+
86+
- name: Create test summary from report
87+
if: always()
7488
run: |
75-
echo "=== Kind Cluster Info ==="
76-
kind get clusters || true
77-
kubectl cluster-info --context kind-semantic-router-e2e || true
78-
79-
echo "=== All Pods ==="
80-
kubectl get pods --all-namespaces -o wide || true
81-
82-
echo "=== Semantic Router Logs ==="
83-
kubectl logs -n vllm-semantic-router-system deployment/semantic-router --tail=100 || true
84-
85-
echo "=== Envoy Gateway Logs ==="
86-
kubectl logs -n envoy-gateway-system deployment/envoy-gateway --tail=100 || true
87-
88-
echo "=== AI Gateway Controller Logs ==="
89-
kubectl logs -n envoy-ai-gateway-system deployment/ai-gateway-controller --tail=100 || true
90-
91-
echo "=== Gateway Resources ==="
92-
kubectl get gateway -A || true
93-
kubectl get httproute -A || true
94-
95-
echo "=== Events ==="
96-
kubectl get events --all-namespaces --sort-by='.lastTimestamp' || true
89+
if [ -f "test-report.md" ]; then
90+
echo "=== Reading test report from test-report.md ==="
91+
cat test-report.md >> $GITHUB_STEP_SUMMARY
92+
93+
# Add additional context
94+
cat >> $GITHUB_STEP_SUMMARY << 'EOF'
95+
96+
---
97+
98+
### 📚 Additional Resources
99+
100+
- **Trigger:** ${{ github.event_name }}
101+
- **Branch:** `${{ github.ref_name }}`
102+
- **Commit:** `${{ github.sha }}`
103+
- **Workflow Run:** [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
104+
- [E2E Test Framework Documentation](https://github.com/${{ github.repository }}/tree/main/e2e)
105+
- [AI Gateway Profile](https://github.com/${{ github.repository }}/tree/main/e2e/profiles/ai-gateway)
106+
107+
### � Artifacts
108+
109+
- Test reports (JSON and Markdown) are available as workflow artifacts
110+
- Reports are retained for 30 days
111+
EOF
112+
else
113+
echo "⚠️ Test report file not found!" >> $GITHUB_STEP_SUMMARY
114+
echo "" >> $GITHUB_STEP_SUMMARY
115+
echo "The E2E test framework did not generate a report file." >> $GITHUB_STEP_SUMMARY
116+
echo "This might indicate that the test failed before report generation." >> $GITHUB_STEP_SUMMARY
117+
fi
97118
98119
- name: Clean up
99120
if: always()

‎e2e/pkg/framework/debug.go‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package framework
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
8+
corev1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/client-go/kubernetes"
11+
)
12+
13+
// PrintAllPodsStatus prints detailed status and logs for all pods in all namespaces
14+
// This is useful for debugging when tests fail
15+
func PrintAllPodsStatus(ctx context.Context, client *kubernetes.Clientset) {
16+
fmt.Printf("\n========== All Pods Status and Logs ==========\n")
17+
18+
// Get all pods from all namespaces
19+
pods, err := client.CoreV1().Pods("").List(ctx, metav1.ListOptions{})
20+
if err != nil {
21+
fmt.Printf("Failed to list pods: %v\n", err)
22+
return
23+
}
24+
25+
fmt.Printf("Total pods: %d\n", len(pods.Items))
26+
27+
for _, pod := range pods.Items {
28+
fmt.Printf("\n--- Pod: %s/%s ---\n", pod.Namespace, pod.Name)
29+
fmt.Printf("Phase: %s\n", pod.Status.Phase)
30+
fmt.Printf("Node: %s\n", pod.Spec.NodeName)
31+
fmt.Printf("Start Time: %v\n", pod.Status.StartTime)
32+
33+
// Print conditions
34+
fmt.Printf("\nConditions:\n")
35+
for _, condition := range pod.Status.Conditions {
36+
fmt.Printf(" - Type: %s, Status: %s, Reason: %s, Message: %s\n",
37+
condition.Type, condition.Status, condition.Reason, condition.Message)
38+
}
39+
40+
// Print container statuses
41+
fmt.Printf("\nContainer Statuses:\n")
42+
for _, containerStatus := range pod.Status.ContainerStatuses {
43+
fmt.Printf(" - Container: %s\n", containerStatus.Name)
44+
fmt.Printf(" Ready: %v\n", containerStatus.Ready)
45+
fmt.Printf(" RestartCount: %d\n", containerStatus.RestartCount)
46+
fmt.Printf(" Image: %s\n", containerStatus.Image)
47+
48+
if containerStatus.State.Waiting != nil {
49+
fmt.Printf(" State: Waiting\n")
50+
fmt.Printf(" Reason: %s\n", containerStatus.State.Waiting.Reason)
51+
fmt.Printf(" Message: %s\n", containerStatus.State.Waiting.Message)
52+
} else if containerStatus.State.Running != nil {
53+
fmt.Printf(" State: Running\n")
54+
fmt.Printf(" Started At: %v\n", containerStatus.State.Running.StartedAt)
55+
} else if containerStatus.State.Terminated != nil {
56+
fmt.Printf(" State: Terminated\n")
57+
fmt.Printf(" Reason: %s\n", containerStatus.State.Terminated.Reason)
58+
fmt.Printf(" Message: %s\n", containerStatus.State.Terminated.Message)
59+
fmt.Printf(" Exit Code: %d\n", containerStatus.State.Terminated.ExitCode)
60+
}
61+
62+
if containerStatus.LastTerminationState.Terminated != nil {
63+
fmt.Printf(" Last Termination:\n")
64+
fmt.Printf(" Reason: %s\n", containerStatus.LastTerminationState.Terminated.Reason)
65+
fmt.Printf(" Message: %s\n", containerStatus.LastTerminationState.Terminated.Message)
66+
fmt.Printf(" Exit Code: %d\n", containerStatus.LastTerminationState.Terminated.ExitCode)
67+
}
68+
}
69+
70+
// Print init container statuses if any
71+
if len(pod.Status.InitContainerStatuses) > 0 {
72+
fmt.Printf("\nInit Container Statuses:\n")
73+
for _, containerStatus := range pod.Status.InitContainerStatuses {
74+
fmt.Printf(" - Container: %s\n", containerStatus.Name)
75+
fmt.Printf(" Ready: %v\n", containerStatus.Ready)
76+
fmt.Printf(" RestartCount: %d\n", containerStatus.RestartCount)
77+
78+
if containerStatus.State.Waiting != nil {
79+
fmt.Printf(" State: Waiting - %s: %s\n",
80+
containerStatus.State.Waiting.Reason,
81+
containerStatus.State.Waiting.Message)
82+
} else if containerStatus.State.Running != nil {
83+
fmt.Printf(" State: Running\n")
84+
} else if containerStatus.State.Terminated != nil {
85+
fmt.Printf(" State: Terminated - %s: %s (Exit Code: %d)\n",
86+
containerStatus.State.Terminated.Reason,
87+
containerStatus.State.Terminated.Message,
88+
containerStatus.State.Terminated.ExitCode)
89+
}
90+
}
91+
}
92+
93+
// Print events for this pod
94+
fmt.Printf("\nRecent Events:\n")
95+
events, err := client.CoreV1().Events(pod.Namespace).List(ctx, metav1.ListOptions{
96+
FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.kind=Pod", pod.Name),
97+
})
98+
if err == nil && len(events.Items) > 0 {
99+
// Sort events by last timestamp (most recent first)
100+
for i := len(events.Items) - 1; i >= 0 && i >= len(events.Items)-10; i-- {
101+
event := events.Items[i]
102+
fmt.Printf(" - [%s] %s: %s (Count: %d)\n",
103+
event.LastTimestamp.Format("15:04:05"),
104+
event.Reason,
105+
event.Message,
106+
event.Count)
107+
}
108+
} else if err != nil {
109+
fmt.Printf(" Failed to get events: %v\n", err)
110+
} else {
111+
fmt.Printf(" No events found\n")
112+
}
113+
114+
// Print container logs
115+
fmt.Printf("\nContainer Logs:\n")
116+
for _, container := range pod.Spec.Containers {
117+
fmt.Printf("\n --- Logs for container: %s ---\n", container.Name)
118+
logOptions := &corev1.PodLogOptions{
119+
Container: container.Name,
120+
TailLines: int64Ptr(50), // Last 50 lines
121+
}
122+
123+
req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, logOptions)
124+
logs, err := req.Stream(ctx)
125+
if err != nil {
126+
fmt.Printf(" Failed to get logs: %v\n", err)
127+
continue
128+
}
129+
defer logs.Close()
130+
131+
logBytes, err := io.ReadAll(logs)
132+
if err != nil {
133+
fmt.Printf(" Failed to read logs: %v\n", err)
134+
continue
135+
}
136+
137+
if len(logBytes) == 0 {
138+
fmt.Printf(" (no logs available)\n")
139+
} else {
140+
fmt.Printf("%s\n", string(logBytes))
141+
}
142+
}
143+
144+
// Print init container logs if any failed
145+
for _, containerStatus := range pod.Status.InitContainerStatuses {
146+
if !containerStatus.Ready {
147+
fmt.Printf("\n --- Logs for init container: %s ---\n", containerStatus.Name)
148+
logOptions := &corev1.PodLogOptions{
149+
Container: containerStatus.Name,
150+
TailLines: int64Ptr(50),
151+
}
152+
153+
req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, logOptions)
154+
logs, err := req.Stream(ctx)
155+
if err != nil {
156+
fmt.Printf(" Failed to get logs: %v\n", err)
157+
continue
158+
}
159+
defer logs.Close()
160+
161+
logBytes, err := io.ReadAll(logs)
162+
if err != nil {
163+
fmt.Printf(" Failed to read logs: %v\n", err)
164+
continue
165+
}
166+
167+
if len(logBytes) == 0 {
168+
fmt.Printf(" (no logs available)\n")
169+
} else {
170+
fmt.Printf("%s\n", string(logBytes))
171+
}
172+
}
173+
}
174+
}
175+
fmt.Printf("\n========================================\n\n")
176+
}
177+
178+
// int64Ptr returns a pointer to an int64 value
179+
func int64Ptr(i int64) *int64 {
180+
return &i
181+
}

0 commit comments

Comments
 (0)