Deepscholar-Bench #1730
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| UV_CACHE_DIR: /tmp/.uv-cache | |
| jobs: | |
| # ============================================================================ | |
| # Verification: Linting and Type Checking | |
| # ============================================================================ | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Restore uv cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| uv-${{ runner.os }}- | |
| - name: Run Ruff linter | |
| run: uv run --frozen --no-group vllm ruff check src/ tests/ | |
| - name: Run Ruff formatter check | |
| run: uv run --frozen --no-group vllm ruff format --check src/ tests/ | |
| type-check: | |
| name: Type Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Restore uv cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| uv-${{ runner.os }}- | |
| - name: Run type checker | |
| run: uv run --frozen --no-group vllm ty check src/ alembic/ | |
| # ============================================================================ | |
| # Unit Tests | |
| # ============================================================================ | |
| test: | |
| name: Test (Python ${{ matrix.python-version }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12", "3.13"] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Restore uv cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| uv-${{ runner.os }}-py${{ matrix.python-version }}- | |
| - name: Run unit tests with coverage | |
| run: | | |
| uv run --frozen --no-group vllm pytest tests/ \ | |
| --ignore=tests/integration/ \ | |
| -v \ | |
| --cov=src/olmo_eval \ | |
| --cov-report=xml \ | |
| --cov-report=html \ | |
| --cov-report=term-missing | |
| - name: Generate coverage summary | |
| if: matrix.python-version == '3.12' | |
| run: | | |
| echo "## Coverage Report" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| uv run --frozen --no-group vllm coverage report --format=markdown >> $GITHUB_STEP_SUMMARY || uv run --frozen --no-group vllm coverage report >> $GITHUB_STEP_SUMMARY | |
| echo "\`\`\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Upload coverage HTML report | |
| uses: actions/upload-artifact@v7 | |
| if: matrix.python-version == '3.12' | |
| with: | |
| name: coverage-report | |
| path: htmlcov/ | |
| retention-days: 14 | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| if: matrix.python-version == '3.12' | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./coverage.xml | |
| fail_ci_if_error: false | |
| verbose: true | |
| # ============================================================================ | |
| # Integration Tests (Storage - Docker) | |
| # ============================================================================ | |
| integration-test-storage: | |
| name: Integration Tests (Storage) | |
| runs-on: ubuntu-latest | |
| needs: [lint, type-check] | |
| timeout-minutes: 15 | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_DB: olmo_eval_test | |
| POSTGRES_USER: test | |
| POSTGRES_PASSWORD: test | |
| ports: | |
| - 5433:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| localstack: | |
| image: localstack/localstack:3.0 | |
| env: | |
| SERVICES: s3 | |
| ports: | |
| - 4566:4566 | |
| options: >- | |
| --health-cmd "curl -f http://localhost:4566/_localstack/health" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Restore uv cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} | |
| restore-keys: | | |
| uv-${{ runner.os }}- | |
| - name: Wait for services to be ready | |
| run: | | |
| echo "Waiting for Postgres..." | |
| until pg_isready -h localhost -p 5433 -U test; do sleep 1; done | |
| echo "Postgres is ready!" | |
| echo "Waiting for LocalStack..." | |
| until curl -f http://localhost:4566/_localstack/health; do sleep 1; done | |
| echo "LocalStack is ready!" | |
| - name: Run storage integration tests | |
| env: | |
| CI_SERVICES_AVAILABLE: "true" | |
| # Dummy AWS credentials for LocalStack (not real credentials) | |
| AWS_ACCESS_KEY_ID: "test" | |
| AWS_SECRET_ACCESS_KEY: "test" | |
| AWS_DEFAULT_REGION: "us-east-1" | |
| run: | | |
| uv run --frozen --no-group vllm pytest tests/integration/test_storage.py \ | |
| -v \ | |
| --tb=short | |
| # ============================================================================ | |
| # Integration Tests (GPU) | |
| # ============================================================================ | |
| integration-test-gpu: | |
| name: Integration Tests (GPU) | |
| runs-on: [self-hosted, gpu] | |
| # Only run when explicitly requested via workflow_dispatch | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [lint, type-check, test] | |
| timeout-minutes: 30 | |
| env: | |
| HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Verify GPU availability | |
| run: | | |
| nvidia-smi | |
| uv run --frozen python -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}'); print(f'GPU count: {torch.cuda.device_count()}')" | |
| - name: Run vLLM integration tests | |
| run: | | |
| uv run --frozen pytest tests/integration/test_vllm_provider.py \ | |
| -v \ | |
| --integration \ | |
| --no-docker \ | |
| --tb=short | |
| # ============================================================================ | |
| # Integration Tests (GPU with Docker) | |
| # ============================================================================ | |
| integration-test-docker: | |
| name: Integration Tests (Docker) | |
| runs-on: [self-hosted, gpu, docker] | |
| # Only run when explicitly requested via workflow_dispatch | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: [lint, type-check, test] | |
| timeout-minutes: 45 | |
| env: | |
| HUGGING_FACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Verify Docker and GPU | |
| run: | | |
| docker --version | |
| docker compose version | |
| nvidia-smi | |
| docker run --rm --gpus all nvidia/cuda:12.0-base nvidia-smi | |
| - name: Run vLLM integration tests with Docker | |
| run: | | |
| uv run --frozen pytest tests/integration/test_vllm_provider.py \ | |
| -v \ | |
| --integration \ | |
| --tb=short | |
| - name: Cleanup Docker containers | |
| if: always() | |
| run: | | |
| docker compose -f tests/integration/docker-compose.vllm.yml down --volumes --remove-orphans || true | |
| # ============================================================================ | |
| # Build Check | |
| # ============================================================================ | |
| build: | |
| name: Build Package | |
| runs-on: ubuntu-latest | |
| needs: [lint, type-check] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Build package | |
| run: uv build | |
| - name: Verify package contents | |
| run: | | |
| ls -la dist/ | |
| uv run --frozen --no-group vllm python -m zipfile -l dist/*.whl | head -50 | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: dist | |
| path: dist/ | |
| retention-days: 5 | |
| # ============================================================================ | |
| # All Checks Passed | |
| # ============================================================================ | |
| ci-success: | |
| name: CI Success | |
| runs-on: ubuntu-latest | |
| needs: [lint, type-check, test, build, integration-test-storage] | |
| if: always() | |
| steps: | |
| - name: Check all jobs passed | |
| run: | | |
| if [[ "${{ needs.lint.result }}" != "success" ]] || \ | |
| [[ "${{ needs.type-check.result }}" != "success" ]] || \ | |
| [[ "${{ needs.test.result }}" != "success" ]] || \ | |
| [[ "${{ needs.build.result }}" != "success" ]] || \ | |
| [[ "${{ needs.integration-test-storage.result }}" != "success" ]]; then | |
| echo "One or more required jobs failed" | |
| exit 1 | |
| fi | |
| echo "All CI checks passed!" |