refactor tests

Author: thesofakillers

.github/workflows/paperbench_tests.yml

@@ -5,6 +5,7 @@ on:
     paths: [ 'project/paperbench/**']
     branches: [ '**' ]
     types: [opened, synchronize, reopened, ready_for_review]
+
   workflow_dispatch:
     inputs:
       test_branch:
@@ -43,7 +44,7 @@ jobs:
         driver: docker
         install: true
 
-    - name: Build pb-env (local)
+    - name: Build pb-env
       working-directory: ./project/paperbench
       run: |
         docker buildx build \
@@ -56,4 +57,4 @@ jobs:
     - name: Run tests
       working-directory: ./project/paperbench
       run: |
-        uv run pytest -n 10 -v -rs
+        uv run pytest -n 10 -vrs

project/paperbench/experiments/splits/testing.txt

@@ -0,0 +1 @@
+lca-on-the-line

project/paperbench/paperbench/nano/eval.py

@@ -391,9 +391,9 @@ class PaperBench(PythonCodingEval):
     judge: JudgeConfig = chz.field(default_factory=JudgeConfig)
 
     # task args
-    paper_split: Literal["debug", "dev", "human", "all"] = chz.field(
+    paper_split: Literal["debug", "dev", "human", "testing", "all"] = chz.field(
         default="all",
-        doc="Paper split to use. One of 'debug' (rice only), 'dev' (two papers), 'human' (papers used in human baseline), 'all' (full set)",
+        doc="Paper split to use. One of 'testing' (lca-on-the-line only), 'debug' (rice only), 'dev' (two papers), 'human' (papers used in human baseline), 'all' (full set)",
         # should match what is in experiments/splits/
     )
     resume_run_group_id: str | None = chz.field(default=None)

project/paperbench/paperbench/scripts/alcatraz_services.py

@@ -10,10 +10,10 @@
 
 from paperbench.infra.alcatraz import put_file_in_computer, tar_and_extract_from_computer
 from paperbench.scripts.run_reproduce import reproduce
-from paperbench.utils import get_dotenv
+from paperbench.utils import find_dotenv
 
 logger = structlog.stdlib.get_logger(component=__name__)
-load_dotenv(get_dotenv())
+load_dotenv(find_dotenv())
 
 
 async def put_submission_in_computer(

project/paperbench/paperbench/utils.py

@@ -1,6 +1,7 @@
 import io
 import logging
 import os
+import subprocess
 import tarfile
 import time
 import uuid
@@ -82,7 +83,7 @@ def get_experiments_dir() -> Path:
     return get_root().parent / "experiments"
 
 
-def get_dotenv() -> Path:
+def find_dotenv() -> Path:
     """Returns an absolute path to the .env file."""
 
     return get_root().parent / ".env"
@@ -94,9 +95,13 @@ def get_timestamp() -> str:
     return time.strftime("%Y-%m-%dT%H-%M-%S-%Z", time.gmtime())
 
 
-def create_run_id(
-    paper_id: str,
-) -> str:
+def get_commit_hash() -> str:
+    """Returns the current Git commit hash."""
+
+    return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip().decode("ascii")
+
+
+def create_run_id(paper_id: str) -> str:
     """Creates a run ID."""
 
     return f"{paper_id}_{str(uuid.uuid4())}"

project/paperbench/tests/conftest.py

@@ -4,12 +4,6 @@
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
-    parser.addoption(
-        "--no-cache",
-        action="store_true",
-        default=False,
-        help="Download and prepare all datasets from scratch.",
-    )
     parser.addoption(
         "--slow",
         action="store_true",

project/paperbench/tests/integration/.gitignore

@@ -0,0 +1 @@
+runs

project/paperbench/tests/integration/configs.py

@@ -0,0 +1,73 @@
+import os
+
+from dotenv import load_dotenv
+
+from paperbench.utils import find_dotenv
+
+load_dotenv(find_dotenv())
+
+from alcatraz.clusters.local import LocalConfig
+from preparedness_turn_completer.oai_turn_completer import OpenAITurnCompleter
+
+from paperbench.nano.eval import (
+    ExternalPythonCodingSolver,
+)
+from paperbench.nano.structs import (
+    JudgeConfig,
+    ReproductionConfig,
+)
+from paperbench.solvers.dummy.solver import PaperBenchDummySolver
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", None)
+GRADER_OPENAI_API_KEY = os.getenv("GRADER_OPENAI_API_KEY", OPENAI_API_KEY)
+DEFAULT_AZURE_VM_SKU = "Standard_D2as_v4"
+
+LOCAL_DUMMY_SOLVER_CONFIG = PaperBenchDummySolver()
+
+LOCAL_AISI_SOLVER_CONFIG = ExternalPythonCodingSolver(
+    agent_id="aisi-basic-agent-openai-dev",
+    cluster_config=LocalConfig(
+        image="aisi-basic-agent:latest",
+        pull_from_registry=False,
+    ),
+)
+
+LOCAL_REPRODUCTION_CONFIG = ReproductionConfig(
+    timeout=100 * 3600,
+    retry_threshold=600,
+    overwrite_existing_output=False,
+    skip_reproduction=True,
+    cluster_config=LocalConfig(
+        image="pb-reproducer:latest",
+        pull_from_registry=False,
+    ),
+)
+
+LOCAL_REPRODUCTION_WITHOUT_SKIP_CONFIG = ReproductionConfig(
+    timeout=100 * 3600,
+    retry_threshold=600,
+    overwrite_existing_output=False,
+    skip_reproduction=False,
+    cluster_config=LocalConfig(
+        image="pb-reproducer:latest",
+        pull_from_registry=False,
+    ),
+)
+
+LOCAL_JUDGE_CONFIG = JudgeConfig(
+    grade=True,
+    grade_locally=True,
+    grade_id=0,
+    overwrite_existing_output=False,
+    scaffold="dummy",
+    completer_config=OpenAITurnCompleter.Config(
+        model="gpt-4o-mini",
+    ),
+    code_only=False,
+    resources_provided=False,
+    cluster_config=LocalConfig(
+        image="pb-env:latest",
+        pull_from_registry=False,
+        environment={"OPENAI_API_KEY": GRADER_OPENAI_API_KEY},
+    ),
+)