Test Tutorial Agents #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Tutorial Agents | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| test-tutorials: | |
| timeout-minutes: 15 | |
| name: test-tutorial-${{ matrix.tutorial }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout agentex-python repo | |
| uses: actions/checkout@v4 | |
| - name: Install UV | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Checkout scale-agentex repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: scaleapi/scale-agentex | |
| path: scale-agentex | |
| - name: Navigate to scale-agentex repo | |
| run: | | |
| cd scale-agentex/agentex | |
| echo "🚀 Starting dependencies (Postgres, Redis, Temporal, MongoDB)..." | |
| # Start all services except the agentex service | |
| docker compose up -d | |
| echo "⏳ Waiting for dependencies to be healthy..." | |
| # Wait for services to be healthy | |
| for i in {1..30}; do | |
| if docker compose ps | grep -q "healthy"; then | |
| echo "✅ Dependencies are healthy" | |
| break | |
| fi | |
| echo " Attempt $i/30: Waiting for services..." | |
| sleep 5 | |
| done | |
| # Verify all deps are up | |
| docker compose ps | |
| - name: Build AgentEx SDK | |
| run: | | |
| echo "🔨 Building AgentEx SDK wheel..." | |
| uv build | |
| echo "✅ SDK built successfully" | |
| ls -la dist/ | |
| - name: Run Parallel Tutorial Tests | |
| working-directory: ./examples/tutorials | |
| run: | | |
| # Verify uv is working | |
| echo "UV version: $(uv --version)" | |
| echo "UV path: $(which uv)" | |
| # Start background job to continuously poll AgentEx container logs | |
| echo "🔍 Starting AgentEx container log polling in background..." | |
| ( | |
| cd ../scale-agentex/agentex | |
| echo "Log polling started at $(date)" | |
| # Function to get container logs | |
| get_logs() { | |
| echo "=== AgentEx Container Logs $(date) ===" | |
| if docker compose logs agentex --tail=20 2>/dev/null; then | |
| echo "✅ Successfully retrieved agentex logs" | |
| elif docker logs $(docker ps --filter "name=agentex" --format "{{.ID}}" 2>/dev/null) --tail=20 2>/dev/null; then | |
| echo "✅ Retrieved logs via container ID" | |
| else | |
| echo "⚠️ No agentex container logs available" | |
| docker compose ps 2>/dev/null || echo "Docker compose ps failed" | |
| fi | |
| echo "=== End Logs ===" | |
| echo "" | |
| } | |
| # Poll logs every 30 seconds | |
| while true; do | |
| get_logs | |
| sleep 30 | |
| done | |
| ) & | |
| LOG_POLLER_PID=$! | |
| echo "📋 Log poller started with PID: $LOG_POLLER_PID" | |
| # Find all tutorial directories | |
| tutorial_paths=() | |
| for dir in $(find . -name "manifest.yaml" -exec dirname {} \; | sort); do | |
| tutorial_paths+=("${dir#./}") # Remove leading ./ | |
| done | |
| echo "Found ${#tutorial_paths[@]} tutorials:" | |
| printf ' %s\n' "${tutorial_paths[@]}" | |
| # Run tests in parallel with unique ports | |
| pids=() | |
| failed_tests=() | |
| passed_tests=() | |
| # Run only the first test for sanity check | |
| for i in 0; do | |
| tutorial="${tutorial_paths[$i]}" | |
| port=$((8000 + i)) | |
| echo "" | |
| echo "=========================================" | |
| echo "Starting test $((i+1))/${#tutorial_paths[@]}: $tutorial (port $port)" | |
| echo "=========================================" | |
| # Modify manifest.yaml to use unique port | |
| manifest_path="$tutorial/manifest.yaml" | |
| if [ -f "$manifest_path" ]; then | |
| # Backup original manifest | |
| cp "$manifest_path" "$manifest_path.backup" | |
| # Update port in manifest (modify the line containing 'port: 8000' or 'port: XXXX') | |
| sed -i "s/port: [0-9]*/port: $port/" "$manifest_path" | |
| echo "Updated $manifest_path to use port $port" | |
| fi | |
| # Run test in background with unique port | |
| ( | |
| AGENTEX_API_BASE_URL="http://localhost:5003" \ | |
| ./run_agent_test.sh --build-cli "$tutorial" | |
| if [ $? -eq 0 ]; then | |
| echo "✅ PASSED: $tutorial (port $port)" | |
| echo "$tutorial" > "/tmp/passed_$i.txt" | |
| else | |
| echo "❌ FAILED: $tutorial (port $port)" | |
| echo "$tutorial" > "/tmp/failed_$i.txt" | |
| fi | |
| ) & | |
| pids+=($!) | |
| done | |
| # Wait for all tests to complete | |
| echo "" | |
| echo "Waiting for all tests to complete..." | |
| for pid in "${pids[@]}"; do | |
| wait "$pid" | |
| done | |
| # Always show AgentEx server container logs immediately after tests complete | |
| echo "" | |
| echo "=========================================" | |
| echo "AGENTEX SERVER CONTAINER LOGS" | |
| echo "=========================================" | |
| # Show AgentEx server container logs | |
| echo "📋 AgentEx server container logs:" | |
| echo "----------------------------------------" | |
| cd ../scale-agentex/agentex || { | |
| echo "❌ Failed to navigate to scale-agentex/agentex directory" | |
| echo "Current directory: $(pwd)" | |
| echo "Directory contents:" | |
| ls -la ../ | |
| } | |
| echo "Current directory: $(pwd)" | |
| echo "Available containers:" | |
| docker compose ps || docker ps | |
| echo "" | |
| echo "AgentEx container logs (last 100 lines):" | |
| docker compose logs agentex --tail=100 || { | |
| echo "❌ Failed to get agentex container logs, trying alternative methods..." | |
| docker logs $(docker ps --filter "name=agentex" --format "{{.ID}}") --tail=100 2>/dev/null || { | |
| echo "❌ No agentex container found, showing all container logs:" | |
| docker compose logs --tail=50 || { | |
| echo "❌ Docker compose logs failed, trying docker logs for all containers:" | |
| docker ps --format "table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Names}}" | |
| for container in $(docker ps --format "{{.ID}}"); do | |
| echo "--- Container $container logs ---" | |
| docker logs "$container" --tail=20 | |
| done | |
| } | |
| } | |
| } | |
| echo "----------------------------------------" | |
| # Return to tutorial directory | |
| cd ../../examples/tutorials | |
| # Stop the log poller | |
| echo "🛑 Stopping log poller (PID: $LOG_POLLER_PID)" | |
| kill $LOG_POLLER_PID 2>/dev/null || echo "Log poller already stopped" | |
| # Restore all original manifests | |
| echo "" | |
| echo "Restoring original manifest files..." | |
| for tutorial in "${tutorial_paths[@]}"; do | |
| if [ -f "$tutorial/manifest.yaml.backup" ]; then | |
| mv "$tutorial/manifest.yaml.backup" "$tutorial/manifest.yaml" | |
| echo "Restored $tutorial/manifest.yaml" | |
| fi | |
| done | |
| # Collect results | |
| for i in "${!tutorial_paths[@]}"; do | |
| if [ -f "/tmp/passed_$i.txt" ]; then | |
| passed_tests+=($(cat "/tmp/passed_$i.txt")) | |
| elif [ -f "/tmp/failed_$i.txt" ]; then | |
| failed_tests+=($(cat "/tmp/failed_$i.txt")) | |
| fi | |
| done | |
| # Print summary | |
| echo "" | |
| echo "=========================================" | |
| echo "TEST SUMMARY" | |
| echo "=========================================" | |
| echo "Total: ${#tutorial_paths[@]}" | |
| echo "Passed: ${#passed_tests[@]}" | |
| echo "Failed: ${#failed_tests[@]}" | |
| if [ ${#failed_tests[@]} -gt 0 ]; then | |
| echo "" | |
| echo "Failed tests:" | |
| for test in "${failed_tests[@]}"; do | |
| echo " ❌ $test" | |
| done | |
| echo "" | |
| echo "=========================================" | |
| echo "TUTORIAL AGENT LOGS FOR FAILED TESTS" | |
| echo "=========================================" | |
| # Show logs for failed tests | |
| for test in "${failed_tests[@]}"; do | |
| test_name=$(basename "$test") | |
| logfile="/tmp/agentex-${test_name}.log" | |
| echo "" | |
| echo "📋 Tutorial agent logs for $test ($logfile):" | |
| echo "----------------------------------------" | |
| if [ -f "$logfile" ]; then | |
| echo "Last 100 lines of tutorial agent logs:" | |
| tail -100 "$logfile" | |
| else | |
| echo "⚠️ No log file found at: $logfile" | |
| echo "Available log files:" | |
| ls -la /tmp/agentex-*.log 2>/dev/null || echo " (none)" | |
| fi | |
| echo "----------------------------------------" | |
| done | |
| exit 1 | |
| else | |
| echo "" | |
| echo "🎉 All tests passed!" | |
| fi |