Test Tutorial Agents #59
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| find-tutorials: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tutorials: ${{ steps.get-tutorials.outputs.tutorials }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Find all tutorials | |
| id: get-tutorials | |
| run: | | |
| cd examples/tutorials | |
| # Find all tutorials and exclude specific temporal ones | |
| all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||') | |
| # Filter out the specified temporal tutorials that are being updated | |
| filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)") | |
| # Convert to JSON array | |
| tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "tutorials=$tutorials" >> $GITHUB_OUTPUT | |
| echo "All tutorials found: $(echo "$all_tutorials" | wc -l)" | |
| echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)" | |
| echo "Excluded tutorials:" | |
| echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo " (none matched exclusion pattern)" | |
| echo "Final tutorial list: $tutorials" | |
| test-tutorial: | |
| needs: find-tutorials | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| matrix: | |
| tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }} | |
| fail-fast: false | |
| name: test-${{ matrix.tutorial }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Install UV | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Checkout scale-agentex repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scaleapi/scale-agentex | |
| path: scale-agentex | |
| - name: Configure Docker Compose for host networking | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🔧 Configuring AgentEx container for GitHub Actions networking..." | |
| # Install yq for YAML manipulation | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| # Add extra_hosts to agentex service to make host.docker.internal work | |
| yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml | |
| echo "✅ Added extra_hosts configuration to agentex service" | |
| - name: Start AgentEx Server | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🚀 Starting AgentEx server and dependencies..." | |
| # Start all services | |
| docker compose up -d | |
| echo "⏳ Waiting for dependencies to be healthy..." | |
| # Wait for services to be healthy | |
| for i in {1..30}; do | |
| if docker compose ps | grep -q "healthy"; then | |
| echo "✅ Dependencies are healthy" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for services..." | |
| sleep 5 | |
| done | |
| # Wait specifically for AgentEx server to be ready | |
| echo "⏳ Waiting for AgentEx server to be ready..." | |
| for i in {1..30}; do | |
| if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then | |
| echo "✅ AgentEx server is ready" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for AgentEx server..." | |
| sleep 5 | |
| done | |
| - name: Build AgentEx SDK | |
| run: | | |
| echo "🔨 Building AgentEx SDK wheel..." | |
| uv build | |
| echo "✅ SDK built successfully" | |
| ls -la dist/ | |
| - name: Test Tutorial | |
| working-directory: ./examples/tutorials | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }} | |
| HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks | |
| run: | | |
| echo "Testing tutorial: ${{ matrix.tutorial }}" | |
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | |
| ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" | |
| - name: Record test result | |
| id: test-result | |
| if: always() | |
| run: | | |
| if [ "${{ steps.run-test.outcome }}" == "success" ]; then | |
| echo "result=passed" >> $GITHUB_OUTPUT | |
| else | |
| echo "result=failed" >> $GITHUB_OUTPUT | |
| fi | |
| test-summary: | |
| if: always() | |
| needs: [find-tutorials, test-tutorial] | |
| runs-on: ubuntu-latest | |
| name: Test Summary | |
| steps: | |
| - name: Generate Test Summary | |
| run: | | |
| echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Get tutorial list from needs context | |
| tutorials='${{ needs.find-tutorials.outputs.tutorials }}' | |
| # Fetch workflow run jobs to get individual test results | |
| jobs_json=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs") | |
| # Initialize counters | |
| total_tutorials=0 | |
| passed_tutorials=0 | |
| failed_tutorials=0 | |
| echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Create temporary files for passed and failed tests | |
| > /tmp/passed_tests.txt | |
| > /tmp/failed_tests.txt | |
| # Process each tutorial | |
| echo "$tutorials" | jq -r '.[]' | while read -r tutorial_name; do | |
| if [ -z "$tutorial_name" ]; then | |
| continue | |
| fi | |
| total_tutorials=$((total_tutorials + 1)) | |
| # Find the job for this tutorial | |
| job_name="test-$tutorial_name" | |
| job_conclusion=$(echo "$jobs_json" | jq -r ".jobs[] | select(.name == \"$job_name\") | .conclusion") | |
| if [ "$job_conclusion" == "success" ]; then | |
| passed_tutorials=$((passed_tutorials + 1)) | |
| echo "$tutorial_name" >> /tmp/passed_tests.txt | |
| echo "✅ $tutorial_name: PASSED" | |
| else | |
| failed_tutorials=$((failed_tutorials + 1)) | |
| echo "$tutorial_name" >> /tmp/failed_tests.txt | |
| echo "❌ $tutorial_name: FAILED (conclusion: $job_conclusion)" | |
| fi | |
| done | |
| # Read the final counts (since the while loop runs in a subshell) | |
| passed_tutorials=$(wc -l < /tmp/passed_tests.txt | tr -d ' ') | |
| failed_tutorials=$(wc -l < /tmp/failed_tests.txt | tr -d ' ') | |
| total_tutorials=$((passed_tutorials + failed_tutorials)) | |
| echo "Final counts: total=$total_tutorials, passed=$passed_tutorials, failed=$failed_tutorials" | |
| # Show summary stats | |
| echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Show passed tests | |
| if [ $passed_tutorials -gt 0 ]; then | |
| echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| while read -r test; do | |
| if [ -n "$test" ]; then | |
| echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done < /tmp/passed_tests.txt | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show failed tests | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Check the individual job logs for failure details." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| while read -r test; do | |
| if [ -n "$test" ]; then | |
| echo "- ❌ \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done < /tmp/failed_tests.txt | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Set exit code based on results | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "❌ Some tutorials failed. Check the individual job logs for details." >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY | |
| fi |