Test Tutorial Agents #53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| find-tutorials: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tutorials: ${{ steps.get-tutorials.outputs.tutorials }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Find all tutorials | |
| id: get-tutorials | |
| run: | | |
| cd examples/tutorials | |
| # Find all tutorials and exclude specific temporal ones | |
| all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||') | |
| # Filter out the specified temporal tutorials that are being updated | |
| filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)") | |
| # Convert to JSON array | |
| tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "tutorials=$tutorials" >> $GITHUB_OUTPUT | |
| echo "All tutorials found: $(echo "$all_tutorials" | wc -l)" | |
| echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)" | |
| echo "Excluded tutorials:" | |
| echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo " (none matched exclusion pattern)" | |
| echo "Final tutorial list: $tutorials" | |
| test-tutorial: | |
| needs: find-tutorials | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| matrix: | |
| tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }} | |
| fail-fast: false | |
| name: test-${{ matrix.tutorial }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Install UV | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Checkout scale-agentex repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scaleapi/scale-agentex | |
| path: scale-agentex | |
| - name: Configure Docker Compose for host networking | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🔧 Configuring AgentEx container for GitHub Actions networking..." | |
| # Install yq for YAML manipulation | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| # Add extra_hosts to agentex service to make host.docker.internal work | |
| yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml | |
| echo "✅ Added extra_hosts configuration to agentex service" | |
| - name: Start AgentEx Server | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🚀 Starting AgentEx server and dependencies..." | |
| # Start all services | |
| docker compose up -d | |
| echo "⏳ Waiting for dependencies to be healthy..." | |
| # Wait for services to be healthy | |
| for i in {1..30}; do | |
| if docker compose ps | grep -q "healthy"; then | |
| echo "✅ Dependencies are healthy" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for services..." | |
| sleep 5 | |
| done | |
| # Wait specifically for AgentEx server to be ready | |
| echo "⏳ Waiting for AgentEx server to be ready..." | |
| for i in {1..30}; do | |
| if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then | |
| echo "✅ AgentEx server is ready" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for AgentEx server..." | |
| sleep 5 | |
| done | |
| - name: Build AgentEx SDK | |
| run: | | |
| echo "🔨 Building AgentEx SDK wheel..." | |
| uv build | |
| echo "✅ SDK built successfully" | |
| ls -la dist/ | |
| - name: Test Tutorial | |
| working-directory: ./examples/tutorials | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }} | |
| HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks | |
| run: | | |
| echo "Testing tutorial: ${{ matrix.tutorial }}" | |
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | |
| ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" | |
| - name: Upload Test Results | |
| if: always() | |
| run: | | |
| # Sanitize tutorial name for artifact upload | |
| SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g') | |
| echo "Uploading test results for: ${{ matrix.tutorial }} (as: test-results-$SANITIZED_NAME)" | |
| # Create a temporary directory with the sanitized name | |
| mkdir -p "test-results-$SANITIZED_NAME" | |
| cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No log files to copy" | |
| # Upload using the actions/upload-artifact action | |
| echo "artifact-name=test-results-$SANITIZED_NAME" >> $GITHUB_ENV | |
| - name: Upload Artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.artifact-name }} | |
| path: test-results-* | |
| retention-days: 1 | |
| test-summary: | |
| if: always() | |
| needs: [find-tutorials, test-tutorial] | |
| runs-on: ubuntu-latest | |
| name: Test Summary | |
| steps: | |
| - name: Download All Test Results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-results | |
| pattern: test-results-* | |
| - name: Generate Test Summary | |
| run: | | |
| echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Get tutorial list from needs context | |
| tutorials='${{ needs.find-tutorials.outputs.tutorials }}' | |
| # Debug: Show what we're working with | |
| echo "🔍 DEBUG: Tutorial list from find-tutorials job:" | |
| echo "$tutorials" | |
| echo "" | |
| echo "🔍 DEBUG: Downloaded artifacts:" | |
| ls -la test-results/ || echo "No test-results directory found" | |
| echo "" | |
| echo "🔍 DEBUG: Artifact contents:" | |
| find test-results/ -type f -name "*.log" || echo "No log files found" | |
| echo "" | |
| # Initialize counters | |
| total_tutorials=0 | |
| passed_tutorials=0 | |
| failed_tutorials=0 | |
| # Arrays to track results | |
| passed_tests=() | |
| failed_tests=() | |
| echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Process each tutorial result | |
| for tutorial_dir in test-results/test-results-*/; do | |
| if [ -d "$tutorial_dir" ]; then | |
| # Extract sanitized name and convert back to original tutorial path | |
| sanitized_name=$(basename "$tutorial_dir" | sed 's/test-results-//') | |
| tutorial_name=$(echo "$sanitized_name" | sed 's/-/\//g') | |
| total_tutorials=$((total_tutorials + 1)) | |
| # Check if there are any log files in this directory | |
| if find "$tutorial_dir" -name "*.log" -type f | grep -q .; then | |
| # Determine success/failure based on presence of error logs or patterns | |
| if find "$tutorial_dir" -name "*.log" -exec grep -l "FAILED\|ERROR\|Traceback" {} \; | head -1 >/dev/null; then | |
| failed_tutorials=$((failed_tutorials + 1)) | |
| failed_tests+=("$tutorial_name") | |
| else | |
| passed_tutorials=$((passed_tutorials + 1)) | |
| passed_tests+=("$tutorial_name") | |
| fi | |
| fi | |
| fi | |
| done | |
| # Show summary stats | |
| echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Show passed tests | |
| if [ $passed_tutorials -gt 0 ]; then | |
| echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| for test in "${passed_tests[@]}"; do | |
| echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show failed tests with all log contents appended together | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| # Append all failed test log contents together | |
| for test in "${failed_tests[@]}"; do | |
| # Find the log file for this test (convert back to sanitized name) | |
| sanitized_test_name=$(echo "$test" | sed 's/\//-/g') | |
| log_file=$(find "test-results/test-results-$sanitized_test_name" -name "*.log" | head -1) | |
| if [ -f "$log_file" ]; then | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "FAILED: $test" >> $GITHUB_STEP_SUMMARY | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| cat "$log_file" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "FAILED: $test (No log file found)" >> $GITHUB_STEP_SUMMARY | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Set exit code based on results | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY | |
| fi |