Skip to content

Commit 3f59e44

Browse files
RoxyFarhadRoxyFarhad
committed
adding better messages
1 parent c039602 commit 3f59e44

File tree

1 file changed

+157
-1
lines changed

1 file changed

+157
-1
lines changed

‎.github/workflows/agentex-tutorials-test.yml‎

Lines changed: 157 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ jobs:
120120
ls -la dist/
121121
122122
- name: Test Tutorial
123+
id: run-test
123124
working-directory: ./examples/tutorials
124125
env:
125126
OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }}
@@ -129,26 +130,181 @@ jobs:
129130
AGENTEX_API_BASE_URL="http://localhost:5003" \
130131
./run_agent_test.sh --build-cli "${{ matrix.tutorial }}"
131132
133+
- name: Print agent logs on failure
134+
if: failure()
135+
working-directory: ./examples/tutorials
136+
run: |
137+
echo "🚨 Test failed for tutorial: ${{ matrix.tutorial }}"
138+
echo "📋 Printing agent logs..."
139+
140+
# Look for agent log files in the tutorial directory
141+
if find "${{ matrix.tutorial }}" -name "*.log" -type f 2>/dev/null | grep -q .; then
142+
echo "Found agent log files:"
143+
find "${{ matrix.tutorial }}" -name "*.log" -type f -exec echo "=== {} ===" \; -exec cat {} \;
144+
else
145+
echo "No .log files found, checking for other common log locations..."
146+
fi
147+
148+
# Check for any output files or dumps
149+
if find "${{ matrix.tutorial }}" -name "agent_output*" -o -name "debug*" -o -name "*.out" 2>/dev/null | grep -q .; then
150+
echo "Found other output files:"
151+
find "${{ matrix.tutorial }}" -name "agent_output*" -o -name "debug*" -o -name "*.out" -exec echo "=== {} ===" \; -exec cat {} \;
152+
fi
153+
154+
# Print the last 50 lines of any python processes that might still be running
155+
echo "🔍 Checking for running python processes..."
156+
ps aux | grep python || echo "No python processes found"
157+
132158
- name: Record test result
133159
id: test-result
134160
if: always()
135161
run: |
162+
# Create results directory
163+
mkdir -p test-results
164+
165+
# Determine result
136166
if [ "${{ steps.run-test.outcome }}" == "success" ]; then
167+
result="passed"
137168
echo "result=passed" >> $GITHUB_OUTPUT
169+
echo "tutorial=${{ matrix.tutorial }}" >> $GITHUB_OUTPUT
138170
else
171+
result="failed"
139172
echo "result=failed" >> $GITHUB_OUTPUT
173+
echo "tutorial=${{ matrix.tutorial }}" >> $GITHUB_OUTPUT
140174
fi
141175
176+
# Save result to file for artifact upload
177+
# Create a safe filename from tutorial path
178+
safe_name=$(echo "${{ matrix.tutorial }}" | tr '/' '_' | tr -d ' ')
179+
echo "$result" > "test-results/result-${safe_name}.txt"
180+
echo "${{ matrix.tutorial }}" > "test-results/tutorial-${safe_name}.txt"
181+
182+
- name: Upload test result
183+
if: always()
184+
uses: actions/upload-artifact@v4
185+
with:
186+
name: test-result-${{ strategy.job-index }}
187+
path: test-results/
188+
retention-days: 1
189+
142190
test-summary:
143191
if: always()
144192
needs: [find-tutorials, test-tutorial]
145193
runs-on: ubuntu-latest
146194
name: Test Summary
147195
steps:
196+
- name: Download all test results
197+
uses: actions/download-artifact@v4
198+
with:
199+
pattern: test-result-*
200+
path: all-results/
201+
merge-multiple: true
202+
continue-on-error: true
203+
148204
- name: Generate Test Summary
149205
run: |
150206
echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY
151207
echo "" >> $GITHUB_STEP_SUMMARY
152208
153-
# Get tutorial list from needs context
209+
# Initialize counters
210+
passed_count=0
211+
failed_count=0
212+
skipped_count=0
213+
total_count=0
214+
215+
# Get all tutorials that were supposed to run
154216
tutorials='${{ needs.find-tutorials.outputs.tutorials }}'
217+
218+
if [ -d "all-results" ] && [ "$(ls -A all-results 2>/dev/null)" ]; then
219+
echo "📊 Processing individual test results from artifacts..."
220+
221+
echo "## Test Results" >> $GITHUB_STEP_SUMMARY
222+
echo "" >> $GITHUB_STEP_SUMMARY
223+
echo "| Tutorial | Status | Result |" >> $GITHUB_STEP_SUMMARY
224+
echo "|----------|--------|--------|" >> $GITHUB_STEP_SUMMARY
225+
226+
# Process each result file
227+
for result_file in all-results/result-*.txt; do
228+
if [ -f "$result_file" ]; then
229+
# Extract the safe name from filename
230+
safe_name=$(basename "$result_file" .txt | sed 's/result-//')
231+
232+
# Get corresponding tutorial name file
233+
tutorial_file="all-results/tutorial-${safe_name}.txt"
234+
235+
if [ -f "$tutorial_file" ]; then
236+
tutorial_name=$(cat "$tutorial_file")
237+
result=$(cat "$result_file")
238+
239+
total_count=$((total_count + 1))
240+
241+
if [ "$result" = "passed" ]; then
242+
echo "| \`$tutorial_name\` | ✅ | Passed |" >> $GITHUB_STEP_SUMMARY
243+
passed_count=$((passed_count + 1))
244+
else
245+
echo "| \`$tutorial_name\` | ❌ | Failed |" >> $GITHUB_STEP_SUMMARY
246+
failed_count=$((failed_count + 1))
247+
fi
248+
fi
249+
fi
250+
done
251+
252+
# Check for any tutorials that didn't have results (skipped/cancelled)
253+
echo "$tutorials" | jq -r '.[]' | while read expected_tutorial; do
254+
safe_expected=$(echo "$expected_tutorial" | tr '/' '_' | tr -d ' ')
255+
if [ ! -f "all-results/result-${safe_expected}.txt" ]; then
256+
echo "| \`$expected_tutorial\` | ⏭️ | Skipped/Cancelled |" >> $GITHUB_STEP_SUMMARY
257+
skipped_count=$((skipped_count + 1))
258+
total_count=$((total_count + 1))
259+
fi
260+
done
261+
262+
else
263+
echo "⚠️ No individual test results found. This could mean:"
264+
echo "- Test jobs were cancelled before completion"
265+
echo "- Artifacts failed to upload"
266+
echo "- No tutorials were found to test"
267+
echo ""
268+
269+
overall_result="${{ needs.test-tutorial.result }}"
270+
echo "Overall job status: **$overall_result**"
271+
272+
if [[ "$overall_result" == "success" ]]; then
273+
echo "✅ All tests appear to have passed based on job status."
274+
elif [[ "$overall_result" == "failure" ]]; then
275+
echo "❌ Some tests appear to have failed based on job status."
276+
echo ""
277+
echo "💡 **Tip:** Check individual job logs for specific failure details."
278+
elif [[ "$overall_result" == "cancelled" ]]; then
279+
echo "⏭️ Tests were cancelled."
280+
else
281+
echo "❓ Test status is unclear: $overall_result"
282+
fi
283+
284+
# Don't show detailed breakdown when we don't have individual results
285+
tutorial_count=$(echo "$tutorials" | jq -r '. | length')
286+
echo ""
287+
echo "Expected tutorial count: $tutorial_count"
288+
fi
289+
290+
# Only show detailed statistics if we have individual results
291+
if [ -d "all-results" ] && [ "$(ls -A all-results 2>/dev/null)" ]; then
292+
echo "" >> $GITHUB_STEP_SUMMARY
293+
echo "## Summary Statistics" >> $GITHUB_STEP_SUMMARY
294+
echo "" >> $GITHUB_STEP_SUMMARY
295+
echo "- **Total Tests:** $total_count" >> $GITHUB_STEP_SUMMARY
296+
echo "- **Passed:** $passed_count ✅" >> $GITHUB_STEP_SUMMARY
297+
echo "- **Failed:** $failed_count ❌" >> $GITHUB_STEP_SUMMARY
298+
echo "- **Skipped:** $skipped_count ⏭️" >> $GITHUB_STEP_SUMMARY
299+
echo "" >> $GITHUB_STEP_SUMMARY
300+
301+
if [ $failed_count -eq 0 ] && [ $passed_count -gt 0 ]; then
302+
echo "🎉 **All tests passed!**" >> $GITHUB_STEP_SUMMARY
303+
elif [ $failed_count -gt 0 ]; then
304+
echo "⚠️ **Some tests failed.** Check individual job logs for details." >> $GITHUB_STEP_SUMMARY
305+
echo "" >> $GITHUB_STEP_SUMMARY
306+
echo "💡 **Tip:** Look for the 'Print agent logs on failure' step in failed jobs for debugging information." >> $GITHUB_STEP_SUMMARY
307+
else
308+
echo "ℹ️ **Tests were cancelled or skipped.**" >> $GITHUB_STEP_SUMMARY
309+
fi
310+
fi

0 commit comments

Comments
 (0)