frontier-evals/.github/workflows/paperbench_tests.yml at cdb983f7a03c000111d4fec26e51fdf255d5b06a · openai/frontier-evals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
name: Run PaperBench Tests

on:
  pull_request:
    paths:
      - 'project/paperbench/**'
      - 'project/common/alcatraz/**'
      - 'project/common/nanoeval/**'
      - 'project/common/nanoeval_alcatraz/**'
      - 'project/common/preparedness_turn_completer/**'
    branches: [ '**' ]
    types: [opened, synchronize, reopened, ready_for_review]

  workflow_dispatch:
    inputs:
      test_branch:
        description: 'Branch to run the tests on'
        required: false
        default: 'main'

jobs:
  test:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

    steps:
    - uses: actions/checkout@v4
      with:
        lfs: true
        fetch-depth: 1

    - name: Hydrate PaperBench data
      run: |
        git lfs fetch --include "project/paperbench/data/**" --exclude ""
        git lfs checkout project/paperbench/data

    - name: Install uv
      uses: astral-sh/setup-uv@v5

    - name: Install dependencies
      working-directory: ./project/paperbench
      env:
        UV_GIT_LFS: 1
      run: |
        uv sync

    - name: Set up Docker
      uses: docker/setup-buildx-action@v3

    - uses: docker/setup-buildx-action@v3
      with:
        driver: docker
        install: true

    - name: Build pb-env
      working-directory: ./project/paperbench
      run: |
        docker buildx build \
          --builder default \
          --platform=linux/amd64 \
          --load \
          -t pb-env \
          -f paperbench/Dockerfile.base .

    - name: Run tests
      working-directory: ./project/paperbench
      run: |
        uv run pytest -n 10 -vrs