Support --limit option for all tasks

Author: rodneykinney

astabench/evals/inspect_eval_wrappers/core_bench.py

@@ -287,18 +287,18 @@ def score_with_stderr():
 
 
 @task
-def core_bench_test() -> Task:
+def core_bench_test(**kwargs) -> Task:
     """The CORE-bench eval from inspect_evals (test split), using our
     astabench-oriented defaults for the task settings."""
-    return core_bench(split="test")
+    return core_bench(split="test", **kwargs)
 
 
 @task
-def core_bench_validation() -> Task:
+def core_bench_validation(**kwargs) -> Task:
     """The CORE-bench eval from inspect_evals (train split, which we use as
     a 'dev' split), using our astabench-oriented defaults for the task
     settings."""
-    return core_bench(split="train")
+    return core_bench(split="train", **kwargs)
 
 
 __all__ = [

astabench/evals/sqa/task.py

@@ -541,10 +541,10 @@ def sqa(
 
 
 @task
-def sqa_dev() -> Task:
-    return sqa(split="dev")
+def sqa_dev(**kwargs) -> Task:
+    return sqa(split="dev", **kwargs)
 
 
 @task
-def sqa_test() -> Task:
-    return sqa(split="test")
+def sqa_test(**kwargs) -> Task:
+    return sqa(split="test", **kwargs)

astabench/evals/super/task.py

@@ -429,15 +429,15 @@ def super(split: str = "Expert", sample_limit: int | None = None) -> Task:
 
 
 @task
-def super_validation() -> Task:
+def super_validation(limit: int = 50) -> Task:
     """Runs the super task on the validation ("Auto") split."""
     # There are hundreds of samples in "Auto", so we limit it for validation by
     # default; they can only be evaluated if the solver returns trajectory data
     # in the scorer's format
-    return super(split="Auto", sample_limit=50)
+    return super(split="Auto", sample_limit=limit if limit > 0 else None)
 
 
 @task
-def super_test() -> Task:
+def super_test(limit: int = 0) -> Task:
     """Runs the super task on the test ("Expert") split."""
-    return super(split="Expert")
+    return super(split="Expert", sample_limit=limit if limit > 0 else None)

solvers/futurehouse/demo.sh

@@ -4,9 +4,10 @@
 set -euo pipefail
 
 uv run inspect eval \
---solver astabench/solvers/futurehouse/futurehouse_solver.py@futurehouse_solver
+--solver astabench/solvers/futurehouse/futurehouse_solver.py@futurehouse_solver \
 --model openai/gpt-4.1-nano \
 --limit 1 \
+-T limit=1 \
 $* \
 astabench/sqa_dev
 

solvers/react/demo.sh

@@ -10,6 +10,7 @@ uv run astabench eval \
 --solver astabench/solvers/react/basic_agent.py@instantiated_basic_agent \
 --model openai/gpt-4.1-nano \
 --limit 1 \
+-T limit=1 \
 -S max_steps=10 \
 -S with_search_tools=0 -S with_table_editor=0 -S with_report_editor=0 -S with_thinking_tool=0 \
 $*

solvers/sqa/demo.sh

@@ -9,5 +9,6 @@ uv run inspect eval \
 --solver astabench/solvers/sqa/sqa.py@sqa_solver \
 --model openai/gpt-4.1-nano \
 --limit 1 \
+-T limit=1 \
 $* \
 astabench/sqa_dev

solvers/storm/demo.sh

@@ -4,9 +4,10 @@
 set -euo pipefail
 
 uv run inspect eval \
---solver  astabench/solvers/sqa/storm_solver.py@storm_solver \
+--solver astabench/solvers/sqa/storm_solver.py@storm_solver \
 --model openai/gpt-4.1-nano \
 --limit 1 \
+-T limit=1 \
 $* \
 astabench/sqa_dev
 

solvers/super/demo.sh

@@ -7,6 +7,7 @@ uv run inspect eval \
 --solver astabench/solvers/code_agent/agent.py@code_agent \
 --model openai/gpt-4.1-nano \
 --limit 1 \
+-T limit=1 \
 $* \
 astabench/super_validation