Add IBM watsonx AI provider in the list of providers to connect to IBM hosted LLMs #8862
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evals | |
| on: | |
| pull_request: | |
| types: [labeled] | |
| workflow_dispatch: | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| jobs: | |
| evals: | |
| # Run if triggered manually or if PR has 'evals' label. | |
| if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'evals') | |
| runs-on: blacksmith-16vcpu-ubuntu-2404 | |
| timeout-minutes: 45 | |
| defaults: | |
| run: | |
| working-directory: packages/evals | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Create environment | |
| run: | | |
| cat > .env.local << EOF | |
| OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY || 'test-key-for-build' }} | |
| EOF | |
| cat > .env.development << EOF | |
| NODE_ENV=development | |
| DATABASE_URL=postgresql://postgres:password@db:5432/evals_development | |
| REDIS_URL=redis://redis:6379 | |
| HOST_EXECUTION_METHOD=docker | |
| EOF | |
| - name: Build image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: packages/evals/Dockerfile.runner | |
| tags: evals-runner:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| push: false | |
| load: true | |
| - name: Tag image | |
| run: docker tag evals-runner:latest evals-runner | |
| - name: Start containers | |
| run: | | |
| docker compose up -d db redis | |
| timeout 60 bash -c 'until docker compose exec -T db pg_isready -U postgres; do sleep 2; done' | |
| timeout 60 bash -c 'until docker compose exec -T redis redis-cli ping | grep -q PONG; do sleep 2; done' | |
| docker compose run --rm runner sh -c 'nc -z db 5432 && echo "✓ Runner -> Database connection successful"' | |
| docker compose run --rm runner sh -c 'nc -z redis 6379 && echo "✓ Runner -> Redis connection successful"' | |
| docker compose run --rm runner docker ps | |
| - name: Run database migrations | |
| run: docker compose run --rm runner pnpm --filter @roo-code/evals db:migrate | |
| - name: Run evals | |
| run: docker compose run --rm runner pnpm --filter @roo-code/evals cli --ci | |
| - name: Cleanup | |
| if: always() | |
| run: docker compose down -v --remove-orphans |