fixing tests #51
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | ||
|
Check failure on line 1 in .github/workflows/agentex-tutorials-test.yml
|
||
| on: | ||
| workflow_dispatch: | ||
| jobs: | ||
| find-tutorials: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| tutorials: ${{ steps.get-tutorials.outputs.tutorials }} | ||
| steps: | ||
| - name: Checkout agentex-python repo | ||
| uses: actions/checkout@v4 | ||
| - name: Find all tutorials | ||
| id: get-tutorials | ||
| run: | | ||
| cd examples/tutorials | ||
| tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||' | jq -R -s -c 'split("\n") | map(select(length > 0))') | ||
| echo "tutorials=$tutorials" >> $GITHUB_OUTPUT | ||
| echo "Found tutorials: $tutorials" | ||
| test-tutorial: | ||
| needs: find-tutorials | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 15 | ||
| strategy: | ||
| matrix: | ||
| tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }} | ||
| fail-fast: false | ||
| name: test-${{ matrix.tutorial }} | ||
| steps: | ||
| - name: Checkout agentex-python repo | ||
| uses: actions/checkout@v4 | ||
| - name: Install UV | ||
| run: | | ||
| curl -LsSf https://astral.sh/uv/install.sh | sh | ||
| echo "$HOME/.local/bin" >> $GITHUB_PATH | ||
| - name: Checkout scale-agentex repo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: scaleapi/scale-agentex | ||
| path: scale-agentex | ||
| - name: Configure Docker Compose for host networking | ||
| run: | | ||
| cd scale-agentex/agentex | ||
| echo "🔧 Configuring AgentEx container for GitHub Actions networking..." | ||
| # Install yq for YAML manipulation | ||
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | ||
| sudo chmod +x /usr/local/bin/yq | ||
| # Add extra_hosts to agentex service to make host.docker.internal work | ||
| yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml | ||
| echo "✅ Added extra_hosts configuration to agentex service" | ||
| - name: Start AgentEx Server | ||
| run: | | ||
| cd scale-agentex/agentex | ||
| echo "🚀 Starting AgentEx server and dependencies..." | ||
| # Start all services | ||
| docker compose up -d | ||
| echo "⏳ Waiting for dependencies to be healthy..." | ||
| # Wait for services to be healthy | ||
| for i in {1..30}; do | ||
| if docker compose ps | grep -q "healthy"; then | ||
| echo "✅ Dependencies are healthy" | ||
| break | ||
| fi | ||
| echo " Attempt $i/30: Waiting for services..." | ||
| sleep 5 | ||
| done | ||
| # Wait specifically for AgentEx server to be ready | ||
| echo "⏳ Waiting for AgentEx server to be ready..." | ||
| for i in {1..30}; do | ||
| if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then | ||
| echo "✅ AgentEx server is ready" | ||
| break | ||
| fi | ||
| echo " Attempt $i/30: Waiting for AgentEx server..." | ||
| sleep 5 | ||
| done | ||
| - name: Build AgentEx SDK | ||
| run: | | ||
| echo "🔨 Building AgentEx SDK wheel..." | ||
| uv build | ||
| echo "✅ SDK built successfully" | ||
| ls -la dist/ | ||
| - name: Test Tutorial | ||
| working-directory: ./examples/tutorials | ||
| env: | ||
| OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }} | ||
| HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks | ||
| run: | | ||
| echo "Testing tutorial: ${{ matrix.tutorial }}" | ||
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | ||
| ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" | ||
| - name: Upload Test Results | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: test-results-${{ replace(matrix.tutorial, '/', '-') }} | ||
| path: | | ||
| /tmp/agentex-*.log | ||
| retention-days: 1 | ||
| test-summary: | ||
| if: always() | ||
| needs: [find-tutorials, test-tutorial] | ||
| runs-on: ubuntu-latest | ||
| name: Test Summary | ||
| steps: | ||
| - name: Download All Test Results | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| path: test-results | ||
| pattern: test-results-* | ||
| - name: Generate Test Summary | ||
| run: | | ||
| echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Get tutorial list from needs context | ||
| tutorials='${{ needs.find-tutorials.outputs.tutorials }}' | ||
| # Initialize counters | ||
| total_tutorials=0 | ||
| passed_tutorials=0 | ||
| failed_tutorials=0 | ||
| # Arrays to track results | ||
| passed_tests=() | ||
| failed_tests=() | ||
| echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Process each tutorial result | ||
| for tutorial_dir in test-results/test-results-*/; do | ||
| if [ -d "$tutorial_dir" ]; then | ||
| # Extract sanitized name and convert back to original tutorial path | ||
| sanitized_name=$(basename "$tutorial_dir" | sed 's/test-results-//') | ||
| tutorial_name=$(echo "$sanitized_name" | sed 's/-/\//g') | ||
| total_tutorials=$((total_tutorials + 1)) | ||
| # Determine success/failure based on presence of error logs or patterns | ||
| if find "$tutorial_dir" -name "*.log" -exec grep -l "FAILED\|ERROR\|Traceback" {} \; | head -1 >/dev/null; then | ||
| failed_tutorials=$((failed_tutorials + 1)) | ||
| failed_tests+=("$tutorial_name") | ||
| else | ||
| passed_tutorials=$((passed_tutorials + 1)) | ||
| passed_tests+=("$tutorial_name") | ||
| fi | ||
| fi | ||
| done | ||
| # Show summary stats | ||
| echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY | ||
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | ||
| echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Show passed tests | ||
| if [ $passed_tutorials -gt 0 ]; then | ||
| echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| for test in "${passed_tests[@]}"; do | ||
| echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY | ||
| done | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| # Show failed tests with details | ||
| if [ $failed_tutorials -gt 0 ]; then | ||
| echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| for test in "${failed_tests[@]}"; do | ||
| echo "### 🔍 \`$test\`" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Find the log file for this test (convert back to sanitized name) | ||
| sanitized_test_name=$(echo "$test" | sed 's/\//-/g') | ||
| log_file=$(find "test-results/test-results-$sanitized_test_name" -name "*.log" | head -1) | ||
| if [ -f "$log_file" ]; then | ||
| # Extract pytest failures | ||
| if grep -q "FAILED\|ERROR" "$log_file"; then | ||
| echo "**Failed Tests:**" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| grep -A 5 -B 1 "FAILED\|ERROR" "$log_file" | head -20 >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| # Show any Python tracebacks | ||
| if grep -q "Traceback" "$log_file"; then | ||
| echo "**Error Details:**" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| # Get the last traceback in the file | ||
| awk '/Traceback \(most recent call last\)/{p=1} p{print} /^[^ ]/ && p && !/Traceback/{p=0}' "$log_file" | tail -20 >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| else | ||
| echo "_No log file found for detailed error analysis_" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| done | ||
| fi | ||
| # Set exit code based on results | ||
| if [ $failed_tutorials -gt 0 ]; then | ||
| echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY | ||
| exit 1 | ||
| else | ||
| echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
