Test Tutorial Agents #58
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| find-tutorials: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tutorials: ${{ steps.get-tutorials.outputs.tutorials }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Find all tutorials | |
| id: get-tutorials | |
| run: | | |
| cd examples/tutorials | |
| # Find all tutorials and exclude specific temporal ones | |
| all_tutorials=$(find . -name "manifest.yaml" -exec dirname {} \; | sort | sed 's|^\./||') | |
| # Filter out the specified temporal tutorials that are being updated | |
| filtered_tutorials=$(echo "$all_tutorials" | grep -v -E "(temporal)") | |
| # Convert to JSON array | |
| tutorials=$(echo "$filtered_tutorials" | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| echo "tutorials=$tutorials" >> $GITHUB_OUTPUT | |
| echo "All tutorials found: $(echo "$all_tutorials" | wc -l)" | |
| echo "Filtered tutorials: $(echo "$filtered_tutorials" | wc -l)" | |
| echo "Excluded tutorials:" | |
| echo "$all_tutorials" | grep -E "(10_temporal/050_|10_temporal/070_|10_temporal/080_)" || echo " (none matched exclusion pattern)" | |
| echo "Final tutorial list: $tutorials" | |
| test-tutorial: | |
| needs: find-tutorials | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| matrix: | |
| tutorial: ${{ fromJson(needs.find-tutorials.outputs.tutorials) }} | |
| fail-fast: false | |
| name: test-${{ matrix.tutorial }} | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Install UV | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Checkout scale-agentex repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scaleapi/scale-agentex | |
| path: scale-agentex | |
| - name: Configure Docker Compose for host networking | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🔧 Configuring AgentEx container for GitHub Actions networking..." | |
| # Install yq for YAML manipulation | |
| sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 | |
| sudo chmod +x /usr/local/bin/yq | |
| # Add extra_hosts to agentex service to make host.docker.internal work | |
| yq eval '.services.agentex.extra_hosts = ["host.docker.internal:host-gateway"]' -i docker-compose.yml | |
| echo "✅ Added extra_hosts configuration to agentex service" | |
| - name: Start AgentEx Server | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🚀 Starting AgentEx server and dependencies..." | |
| # Start all services | |
| docker compose up -d | |
| echo "⏳ Waiting for dependencies to be healthy..." | |
| # Wait for services to be healthy | |
| for i in {1..30}; do | |
| if docker compose ps | grep -q "healthy"; then | |
| echo "✅ Dependencies are healthy" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for services..." | |
| sleep 5 | |
| done | |
| # Wait specifically for AgentEx server to be ready | |
| echo "⏳ Waiting for AgentEx server to be ready..." | |
| for i in {1..30}; do | |
| if curl -s --max-time 5 http://localhost:5003/health >/dev/null 2>&1; then | |
| echo "✅ AgentEx server is ready" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for AgentEx server..." | |
| sleep 5 | |
| done | |
| - name: Build AgentEx SDK | |
| run: | | |
| echo "🔨 Building AgentEx SDK wheel..." | |
| uv build | |
| echo "✅ SDK built successfully" | |
| ls -la dist/ | |
| - name: Test Tutorial | |
| working-directory: ./examples/tutorials | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.TUTORIAL_OPENAI_API_KEY }} | |
| HEALTH_CHECK_PORT: 8080 # Use non-privileged port for temporal worker health checks | |
| run: | | |
| echo "Testing tutorial: ${{ matrix.tutorial }}" | |
| # Create output files | |
| SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g') | |
| OUTPUT_FILE="/tmp/test-output-$SANITIZED_NAME.txt" | |
| STATUS_FILE="/tmp/test-status-$SANITIZED_NAME.txt" | |
| # Run the test, show output live, and also capture to file | |
| set +e # Don't exit on error | |
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | |
| ./run_agent_test.sh --build-cli "${{ matrix.tutorial }}" 2>&1 | tee "$OUTPUT_FILE" | |
| EXIT_CODE=${PIPESTATUS[0]} # Get exit code from the first command in pipeline | |
| set -e # Re-enable exit on error | |
| # Write the exit code to status file | |
| echo "$EXIT_CODE" > "$STATUS_FILE" | |
| # Exit with the original exit code to make the job succeed/fail correctly | |
| exit $EXIT_CODE | |
| - name: Upload Test Results | |
| if: always() | |
| run: | | |
| # Sanitize tutorial name for artifact upload | |
| SANITIZED_NAME=$(echo "${{ matrix.tutorial }}" | sed 's/\//-/g') | |
| echo "Uploading test results for: ${{ matrix.tutorial }} (as: test-results-$SANITIZED_NAME)" | |
| # Create a temporary directory with the sanitized name | |
| mkdir -p "test-results-$SANITIZED_NAME" | |
| # Copy the test output and status files | |
| cp "/tmp/test-output-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No output file to copy" | |
| cp "/tmp/test-status-$SANITIZED_NAME.txt" "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No status file to copy" | |
| # Also copy agent logs if they exist | |
| cp /tmp/agentex-*.log "test-results-$SANITIZED_NAME/" 2>/dev/null || echo "No agent log files to copy" | |
| # Upload using the actions/upload-artifact action | |
| echo "artifact-name=test-results-$SANITIZED_NAME" >> $GITHUB_ENV | |
| - name: Upload Artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.artifact-name }} | |
| path: test-results-* | |
| retention-days: 1 | |
| test-summary: | |
| if: always() | |
| needs: [find-tutorials, test-tutorial] | |
| runs-on: ubuntu-latest | |
| name: Test Summary | |
| steps: | |
| - name: Download All Test Results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-results | |
| pattern: test-results-* | |
| - name: Generate Test Summary | |
| run: | | |
| echo "# 🧪 Tutorial Tests Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Get tutorial list from needs context | |
| tutorials='${{ needs.find-tutorials.outputs.tutorials }}' | |
| # Initialize counters | |
| total_tutorials=0 | |
| passed_tutorials=0 | |
| failed_tutorials=0 | |
| # Arrays to track results | |
| passed_tests=() | |
| failed_tests=() | |
| echo "## 📊 Overall Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Process each tutorial and check its status file | |
| echo "$tutorials" | jq -r '.[]' | while read -r tutorial_name; do | |
| total_tutorials=$((total_tutorials + 1)) | |
| # Convert tutorial name to sanitized format | |
| sanitized_name=$(echo "$tutorial_name" | sed 's/\//-/g') | |
| tutorial_dir="test-results/test-results-$sanitized_name" | |
| status_file="$tutorial_dir/test-status-$sanitized_name.txt" | |
| if [ -f "$status_file" ]; then | |
| exit_code=$(cat "$status_file") | |
| if [ "$exit_code" = "0" ]; then | |
| passed_tutorials=$((passed_tutorials + 1)) | |
| passed_tests+=("$tutorial_name") | |
| echo "✅ $tutorial_name: PASSED (exit code: $exit_code)" | |
| else | |
| failed_tutorials=$((failed_tutorials + 1)) | |
| failed_tests+=("$tutorial_name") | |
| echo "❌ $tutorial_name: FAILED (exit code: $exit_code)" | |
| fi | |
| else | |
| # No status file found, assume failed | |
| failed_tutorials=$((failed_tutorials + 1)) | |
| failed_tests+=("$tutorial_name") | |
| echo "❌ $tutorial_name: FAILED (no status file)" | |
| fi | |
| done | |
| # Show summary stats | |
| echo "| Status | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ **Passed** | **$passed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ **Failed** | **$failed_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "| 📊 **Total** | **$total_tutorials** |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Show passed tests | |
| if [ $passed_tutorials -gt 0 ]; then | |
| echo "## ✅ Passed Tutorials ($passed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| for test in "${passed_tests[@]}"; do | |
| echo "- ✅ \`$test\`" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Show test output for failed tests | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "## ❌ Failed Tutorials ($failed_tutorials)" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| # Show the full output for each failed test | |
| for test in "${failed_tests[@]}"; do | |
| sanitized_test_name=$(echo "$test" | sed 's/\//-/g') | |
| output_file="test-results/test-results-$sanitized_test_name/test-output-$sanitized_test_name.txt" | |
| if [ -f "$output_file" ]; then | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "FAILED: $test" >> $GITHUB_STEP_SUMMARY | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| cat "$output_file" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "FAILED: $test (No output file found)" >> $GITHUB_STEP_SUMMARY | |
| echo "================================================================================================" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| done | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Set exit code based on results | |
| if [ $failed_tutorials -gt 0 ]; then | |
| echo "❌ Some tutorials failed. Check the details above." >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "🎉 All tutorials passed successfully!" >> $GITHUB_STEP_SUMMARY | |
| fi |