use a eval.yaml from the hub

Author: NathanHB

src/lighteval/__main__.py

@@ -71,7 +71,6 @@
 app.command(rich_help_panel="Evaluation Backends")(lighteval.main_custom.custom)
 app.command(rich_help_panel="Evaluation Backends")(lighteval.main_sglang.sglang)
 app.command(rich_help_panel="Evaluation Backends")(lighteval.main_inspect.eval)
-app.command(rich_help_panel="Evaluation Backends")(lighteval.main_inspect.from_hub)
 app.command(rich_help_panel="EvaluationUtils")(lighteval.main_inspect.bundle)
 app.add_typer(
     lighteval.main_endpoint.app,

src/lighteval/from_hub.py

@@ -14,12 +14,22 @@ def record_to_sample(record, field_spec: dict):
     """
     input_text = record[field_spec["input"]]
 
-    target = record[field_spec["target"]]
+    target = field_spec["target"]
+
+    if target in ascii_uppercase:
+        target = target
+    else:
+        target = record[field_spec["target"]]
 
     if isinstance(target, int):
         target = ascii_uppercase[target]
 
-    choices_list = record[field_spec["choices"]]
+    choices = field_spec["choices"]
+
+    if isinstance(choices, list):
+        choices_list = [record[choice] for choice in choices]
+    else:
+        choices_list = record[choices]
 
     metadata = field_spec.get("metadata", None)
 

src/lighteval/main_inspect.py

@@ -28,7 +28,6 @@
 import requests
 from huggingface_hub import HfApi
 from inspect_ai import Epochs, Task, task
-from inspect_ai import eval as inspect_ai_eval
 from inspect_ai import eval_set as inspect_ai_eval_set
 from inspect_ai.dataset import hf_dataset
 from inspect_ai.log import bundle_log_dir
@@ -215,6 +214,7 @@ def eval(  # noqa C901
     models: Annotated[list[str], Argument(help="Models to evaluate")],
     tasks: Annotated[str, Argument(help="Tasks to evaluate")],
     # model arguments
+    revision: Annotated[str, Option(help="Revision of the benchmark repo on the hub")] = "main",
     model_base_url: Annotated[
         str | None,
         Option(
@@ -430,15 +430,23 @@ def eval(  # noqa C901
         ),
     ] = False,
 ):
+    from huggingface_hub import HfApi
+
     from lighteval.tasks.registry import Registry
 
-    registry = Registry(tasks=tasks, custom_tasks=None, load_multilingual=False)
-    task_configs = registry.task_to_configs
-    inspect_ai_tasks = []
+    if "/" in tasks:
+        api = HfApi()
+        print(f"Loading tasks from dataset repository {tasks}...")
+        api.repo_info(repo_id=tasks, repo_type="dataset", revision=revision)
+        inspect_ai_tasks = create_task_function(tasks, revision)
+    else:
+        registry = Registry(tasks=tasks, custom_tasks=None, load_multilingual=False)
+        task_configs = registry.task_to_configs
+        inspect_ai_tasks = []
 
-    for task_name, task_configs in task_configs.items():
-        for task_config in task_configs:
-            inspect_ai_tasks.append(get_inspect_ai_task(task_config, epochs=epochs, epochs_reducer=epochs_reducer))
+        for task_name, task_configs in task_configs.items():
+            for task_config in task_configs:
+                inspect_ai_tasks.append(get_inspect_ai_task(task_config, epochs=epochs, epochs_reducer=epochs_reducer))
 
     if model_args is not None:
         model_args = InspectAIModelConfig._parse_args(model_args)
@@ -522,12 +530,6 @@ def eval(  # noqa C901
         print("run 'inspect view' to view the results")
 
 
-def from_hub(repo_id: str, models: list[str], limit: int = 100, revision: str = "main"):
-    task = create_task_function(repo_id, revision)
-
-    inspect_ai_eval(tasks=task, model=models, limit=limit)
-
-
 def bundle(log_dir: str, output_dir: str, overwrite: bool = True, repo_id: str | None = None, public: bool = False):
     bundle_log_dir(log_dir=log_dir, output_dir=output_dir, overwrite=overwrite)