nanoeval release

Author: thesofakillers

.github/workflows/swelancer_lint.yml

@@ -0,0 +1,51 @@
+name: Run linting on swelancer
+
+on:
+  pull_request:
+    paths:
+      - 'project/swelancer/**'
+    branches:
+      - '**'
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  workflow_dispatch:
+    inputs:
+      test_branch:
+        description: 'Branch to run the tests on'
+        required: false
+        default: 'main'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        lfs: false
+        fetch-depth: 1
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+
+    - name: Install dependencies
+      working-directory: ./project/swelancer
+      env:
+        UV_GIT_LFS: 1
+      run: |
+        uv sync
+
+    - name: run mypy
+      working-directory: ./project/swelancer
+      run: |
+        uv run poe typecheck
+
+    - name: run ruff-strict
+      working-directory: ./project/swelancer
+      run: |
+        uv run ruff check --output-format=github --no-fix

.pre-commit-config.yaml

@@ -8,23 +8,24 @@ repos:
       # picks up config from [tool.ruff] in pyproject.toml
       - id: ruff-format
         name: ruff-format (black-style)
-        files: (?x)^project/paperbench/
+        files: (?x)^project/(paperbench|swelancer)/
+
       # Lenient hook: runs auto‑fix on all files using a lenient config.
       - id: ruff
-        name: ruff (auto-fix)
+        name: ruff-lenient (auto-fix)
         args:
           - --config=./.ruff_autofix_minimal.toml
           - --unsafe-fixes
           - --exit-non-zero-on-fix
 
       # Strict hook: runs lint only on files opted-in projects (using strict config pulled from pyproject.toml)
       - id: ruff
-        name: ruff (lint)
+        name: ruff-strict (lint)
         args:
           - --fix
           - --fix-only
           - --exit-non-zero-on-fix
-        files: (?x)^project/(alcatraz|nanoeval|paperbench)/
+        files: (?x)^project/(alcatraz|nanoeval|paperbench|swelancer)/
 
   - repo: https://github.com/astral-sh/uv-pre-commit
     # uv version.
@@ -34,5 +35,8 @@ repos:
         name: uv-lock-paperbench
         args: [--project, project/paperbench]
         files: (?x)^project/paperbench/
-
-exclude: 'project/paperbench/paperbench/agents/aisi-basic-agent/inspect_ai'
+      - id: uv-lock
+        name: uv-lock-swelancer
+        args: [--project, project/swelancer]
+        files: (?x)^project/swelancer/
+exclude: '(?x)^(project/paperbench/paperbench/agents/aisi-basic-agent/inspect_ai|project/swelancer/issues)/'

README.md

@@ -9,6 +9,6 @@ Python 3.11 (3.12 is untested; 3.13 will break [chz](https://github.com/openai/c
 ## Evals
 
 - [PaperBench](./project/paperbench/README.md): based on [https://openai.com/index/paperbench/](https://openai.com/index/paperbench/)
-- SWELancer (Forthcoming): based on [https://openai.com/index/swe-lancer/](https://openai.com/index/swe-lancer/)
+- [SWELancer](./project/swelancer/README.md) : based on [https://openai.com/index/swe-lancer/](https://openai.com/index/swe-lancer/)
 - MLE-bench (Forthcoming): based on [https://openai.com/index/mle-bench/](https://openai.com/index/mle-bench/)
 - SWE-bench (Forthcoming): based on [https://openai.com/index/introducing-swe-bench-verified/](https://openai.com/index/introducing-swe-bench-verified/)