Adjust openness and tool usage values (#70)

Author: ca16

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "agent-eval"
-version = "0.1.42"
+version = "0.1.43"
 description = "Agent evaluation toolkit"
 readme = "README.md"
 requires-python = ">=3.10"

src/agenteval/cli.py

@@ -20,7 +20,16 @@
 from agenteval.leaderboard.schema_generator import load_dataset_features
 
 from .cli_utils import AliasedChoice, generate_choice_help
-from .config import load_suite_config
+from .config import (
+    OPENNESS_CLOSED_API_AVAILABLE,
+    OPENNESS_CLOSED_UI_ONLY,
+    OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS,
+    OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS,
+    TOOL_USAGE_CUSTOM_INTERFACE,
+    TOOL_USAGE_FULLY_CUSTOM,
+    TOOL_USAGE_STANDARD,
+    load_suite_config,
+)
 from .io import atomic_write_file
 from .leaderboard.models import LeaderboardSubmission, Readme
 from .leaderboard.upload import (
@@ -39,15 +48,15 @@
 SUBMISSION_METADATA_FILENAME = "submission.json"
 SUMMARIES_PREFIX = "summaries"
 OPENNESS_MAPPING = {
-    "c": "Closed",
-    "api": "API Available",
-    "os": "Open Source",
-    "ow": "Open Source + Open Weights",
+    "c": OPENNESS_CLOSED_UI_ONLY,
+    "api": OPENNESS_CLOSED_API_AVAILABLE,
+    "os": OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS,
+    "ow": OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS,
 }
 TOOL_MAPPING = {
-    "s": "Standard",
-    "css": "Custom with Standard Search",
-    "c": "Fully Custom",
+    "s": TOOL_USAGE_STANDARD,
+    "ci": TOOL_USAGE_CUSTOM_INTERFACE,
+    "c": TOOL_USAGE_FULLY_CUSTOM,
 }
 
 

src/agenteval/config.py

@@ -5,6 +5,19 @@
 import yaml
 from pydantic import BaseModel, ValidationError
 
+# If you change these, be careful about any downstream code
+# that depends on the exact values (e.g. asta-bench-leaderboard
+# expects results to have either these values for openness
+# and tool usage, or values from a specific list of aliases).
+OPENNESS_OPEN_SOURCE_OPEN_WEIGHTS = "Open source & open weights"
+OPENNESS_OPEN_SOURCE_CLOSED_WEIGHTS = "Open source & closed weights"
+OPENNESS_CLOSED_API_AVAILABLE = "Closed source & API available"
+OPENNESS_CLOSED_UI_ONLY = "Closed source & UI only"
+
+TOOL_USAGE_STANDARD = "Standard"
+TOOL_USAGE_CUSTOM_INTERFACE = "Custom interface"
+TOOL_USAGE_FULLY_CUSTOM = "Fully custom"
+
 
 class WeightAdjustment(BaseModel):
     """Weight adjustment for a specific tag-task combination."""