Test Tutorial Agents #52
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| find-tutorials: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tutorials: ${{ steps.get-tutorials.outputs.tutorials }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Find all tutorials | |
| id: get-tutorials | |
| run: | | |
| cd examples/tutorials | |
| tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||' | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "tutorials=$tutorials" >> $GITHUB_OUTPUT | |
| echo "Found tutorials: $tutorials" | |
| test-tutorial: | |
| needs: find-tutorials | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| matrix: | |
| tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }} | |
| fail-fast: false | |
| name: test-${{ matrix.tutorial }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Install UV | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Checkout scale-agentex repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scaleapi/scale-agentex | |
| path: scale-agentex | |
| - name: Configure Docker Compose for host networking | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🔧 Configuring AgentEx container for GitHub Actions networking..." | |
| # Install yq for YAML manipulation | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| # Add extra_hosts to agentex service to make host.docker.internal work | |
| yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml | |
| echo "✅ Added extra_hosts configuration to agentex service" | |
| - name: Start AgentEx Server | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🚀 Starting AgentEx server and dependencies..." | |
| # Start all services | |
| docker compose up -d | |
| echo "⏳ Waiting for dependencies to be healthy..." | |
| # Wait for services to be healthy | |
| for i in {1..30}; do | |
| if docker compose ps | grep -q "healthy"; then | |
| echo "✅ Dependencies are healthy" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for services..." | |
| sleep 5 | |
| done | |
| # Wait specifically for AgentEx server to be ready | |
| echo "⏳ Waiting for AgentEx server to be ready..." | |
| for i in {1..30}; do | |
| if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then | |
| echo "✅ AgentEx server is ready" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for AgentEx server..." | |
| sleep 5 | |
| done | |
| - name: Build AgentEx SDK | |
| run: | | |
| echo "🔨 Building AgentEx SDK wheel..." | |
| uv build | |
| echo "✅ SDK built successfully" | |
| ls -la dist/ | |
| - name: Test Tutorial | |
| working-directory: ./examples/tutorials | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }} | |
| HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks | |
| run: | | |
| echo "Testing tutorial: ${{ matrix.tutorial }}" | |
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | |
| ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" | |
| - name: Upload Test Results | |
| if: always() | |
| run: | | |
| # Sanitize tutorial name for artifact upload | |
| SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g') | |
| echo "Uploading test results for: ${{ matrix.tutorial }} (as: test-results-$SANITIZED_NAME)" | |
| # Create a temporary directory with the sanitized name | |
| mkdir -p "test-results-$SANITIZED_NAME" | |
| cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No log files to copy" | |
| # Upload using the actions/upload-artifact action | |
| echo "artifact-name=test-results-$SANITIZED_NAME" >> $GITHUB_ENV | |
| - name: Upload Artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.artifact-name }} | |
| path: test-results-* | |
| retention-days: 1 | |
| test-summary: | |
| if: always() | |
| needs: [find-tutorials, test-tutorial] | |
| runs-on: ubuntu-latest | |
| name: Test Summary | |
| steps: | |
| - name: Download All Test Results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-results | |
| pattern: test-results-* | |
| - name: Generate Test Summary | |
| run: | | |
| echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Get tutorial list from needs context | |
| tutorials='${{ needs.find-tutorials.outputs.tutorials }}' | |
| # Initialize counters | |
| total_tutorials=0 | |
| passed_tutorials=0 | |
| failed_tutorials=0 | |
| # Arrays to track results | |
| passed_tests=() | |
| failed_tests=() | |
| echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Process each tutorial result | |
| for tutorial_dir in test-results/test-results-*/; do | |
| if [ -d "$tutorial_dir" ]; then | |
| # Extract sanitized name and convert back to original tutorial path | |
| sanitized_name=$(basename "$tutorial_dir" | sed 's/test-results-//') | |
| tutorial_name=$(echo "$sanitized_name" | sed 's/-/\//g') | |
| total_tutorials=$((total_tutorials + 1)) | |
| # Determine success/failure based on presence of error logs or patterns | |
| if find "$tutorial_dir" -name "*.log" -exec grep -l "FAILED\|ERROR\|Traceback" {} \; | head -1 >/dev/null; then | |
| failed_tutorials=$((failed_tutorials + 1)) | |
| failed_tests+=("$tutorial_name") | |
| else | |
| passed_tutorials=$((passed_tutorials + 1)) | |
| passed_tests+=("$tutorial_name") | |
| fi | |
| fi | |
| done | |
| # Show summary stats | |
| echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Show passed tests | |
| if [ $passed_tutorials -gt 0 ]; then | |
| echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| for test in "${passed_tests[@]}"; do | |
| echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show failed tests with details | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| for test in "${failed_tests[@]}"; do | |
| echo "### 🔍 \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Find the log file for this test (convert back to sanitized name) | |
| sanitized_test_name=$(echo "$test" | sed 's/\//-/g') | |
| log_file=$(find "test-results/test-results-$sanitized_test_name" -name "*.log" | head -1) | |
| if [ -f "$log_file" ]; then | |
| # Extract pytest failures | |
| if grep -q "FAILED\|ERROR" "$log_file"; then | |
| echo "**Failed Tests:**" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| grep -A 5 -B 1 "FAILED\|ERROR" "$log_file" | head -20 >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show any Python tracebacks | |
| if grep -q "Traceback" "$log_file"; then | |
| echo "**Error Details:**" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| # Get the last traceback in the file | |
| awk '/Traceback \(most recent call last\)/{p=1} p{print} /^[^ ]/ && p && !/Traceback/{p=0}' "$log_file" | tail -20 >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| else | |
| echo "_No log file found for detailed error analysis_" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done | |
| fi | |
| # Set exit code based on results | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY | |
| fi |