use a eval.yaml from the hub

Author: NathanHB

src/lighteval/from_hub.py

@@ -1,32 +1,10 @@
-import os
-from pathlib import Path
+from importlib import import_module
 from string import ascii_uppercase
 
 import yaml
 from huggingface_hub import hf_hub_download
 from inspect_ai import Epochs, Task, task
 from inspect_ai.dataset import FieldSpec, Sample, hf_dataset
-from inspect_ai.scorer import choice, exact, match, model_graded_fact
-from inspect_ai.solver import (
-    chain_of_thought,
-    generate,
-    multiple_choice,
-    prompt_template,
-    system_message,
-)
-
-
-def load_config(yaml_path: str = None) -> dict:
-    """Load and parse the YAML configuration file."""
-    if yaml_path is None:
-        yaml_path = os.getenv("EVAL_YAML", "eval.yaml")
-
-    yaml_path = Path(yaml_path)
-    if not yaml_path.is_absolute():
-        yaml_path = Path(__file__).parent / yaml_path
-
-    with open(yaml_path, "r") as f:
-        return yaml.safe_load(f)
 
 
 def record_to_sample(record, field_spec: dict):
@@ -51,9 +29,9 @@ def record_to_sample(record, field_spec: dict):
     return Sample(**sample_kwargs)
 
 
-def load_dataset(repo_id: str, revision: str = "main", task_config: dict = None, global_config: dict = None):
+def load_dataset(repo_id: str, revision: str = "main", task_config: dict = None):
     """Load dataset based on task configuration."""
-    subset = task_config.get("subset")
+    subset = task_config.get("subset", "default")
     split = task_config.get("splits", "test")
     field_spec = task_config["field_spec"]
 
@@ -76,85 +54,115 @@ def load_dataset(repo_id: str, revision: str = "main", task_config: dict = None,
             sample_fields=FieldSpec(
                 input=field_spec["input"],
                 target=field_spec["target"],
-                **({k: v for k, v in field_spec.items() if k not in ["input", "target"]}),
+                metadata=field_spec.get("metadata", []),
             ),
         )
 
     return dataset
 
 
 def build_solvers(task_config: dict):
-    """Build solvers list from task configuration."""
+    """
+    Build a list of solvers from the task configuration.
+
+    task_config example:
+
+    ```yaml
+    solvers:
+      - name: prompt_template
+        args:
+          template: >
+            You are a helpful assistant.
+            {prompt}
+      - name: generate
+        args:
+          cache: true
+    ```
+
+
+    """
     solvers = []
-    solver_names = task_config.get("solvers", [])
-
-    for solver_name in solver_names:
-        if solver_name == "prompt_template":
-            if "prompt_template" in task_config and task_config["prompt_template"]:
-                template = task_config["prompt_template"].strip().strip('"')
-                template = template.replace("{{prompt}}", "{prompt}")
-                solvers.append(prompt_template(template))
-        elif solver_name == "system_message":
-            if "system_message" in task_config and task_config["system_message"]:
-                sys_msg = task_config["system_message"].strip().strip('"')
-                solvers.append(system_message(sys_msg))
-        elif solver_name == "chain_of_thought":
-            solvers.append(chain_of_thought())
-        elif solver_name == "multiple_choice":
-            solvers.append(multiple_choice())
-        elif solver_name == "generate":
-            solvers.append(generate())
+    solver_configs = task_config.get("solvers", [])
+    solver_module = import_module("inspect_ai.solver")
 
-    return solvers
+    for solver_config in solver_configs:
+        solver_name = solver_config["name"]
 
+        if not hasattr(solver_module, solver_name):
+            raise ValueError(f"Unknown solver: {solver_name}")
 
-def build_scorer(task_config: dict):
-    """Build scorer from task configuration."""
-    scorer_name = task_config.get("scorers", ["choice"])[0]
-
-    if scorer_name == "choice":
-        return choice()
-    elif scorer_name == "exact":
-        return exact()
-    elif scorer_name == "match":
-        return match()
-    elif scorer_name == "model_graded_fact":
-        return model_graded_fact()
-    else:
-        raise ValueError(f"Unknown scorer: {scorer_name}")
+        solver_fn = getattr(solver_module, solver_name)
+        solvers.append(solver_fn(**solver_config.get("args", {})))
+
+    return solvers
 
 
-def create_task_from_config(
-    repo_id: str, revision: str = "main", task_config: dict = None, global_config: dict = None
-):
+def build_scorer(task_config: dict):
+    """
+    Build a scorer from the task configuration.
+    task_config example:
+
+    ```yaml
+    scorers:
+      - name: model_graded_fact
+        args:
+            template: |
+                grade this,
+
+                question:
+                    {question}
+                criterion:
+                    {criterion}
+                answer:
+                    {answer}
+    ```
+    """
+    scorers = []
+    scorer_configs = task_config.get("scorers", [])
+    scorer_module = import_module("inspect_ai.scorer")
+
+    for scorer_config in scorer_configs:
+        scorer_name = scorer_config["name"]
+
+        if not hasattr(scorer_module, scorer_name):
+            raise ValueError(f"Unknown scorer: {scorer_name}")
+
+        scorer_fn = getattr(scorer_module, scorer_name)
+        scorers.append(scorer_fn(**scorer_config.get("args", {})))
+
+    return scorers
+
+
+@task
+def create_task_from_config(repo_id: str, revision: str = "main", task_config: dict = None):
     """Create an inspect.ai Task from a task configuration."""
-    dataset = load_dataset(repo_id, revision, task_config, global_config)
+    dataset = load_dataset(repo_id, revision, task_config)
     solvers = build_solvers(task_config)
-    scorer = build_scorer(task_config)
+    scorers = build_scorer(task_config)
     epochs = task_config.get("epochs", 1)
     epochs_reducer = task_config.get("epochs_reducer", "mean")
 
     return Task(
         dataset=dataset,
         solver=solvers,
-        scorer=scorer,
+        scorer=scorers,
         name=task_config["name"],
         epochs=Epochs(epochs, epochs_reducer),
     )
 
 
-def create_task_function(repo_id: str, revision: str = "main"):
+def create_task_function(repo_id: str, revision: str = "main") -> list:
     """Factory function to create a task function with proper closure."""
     # read yaml from hf filesystem
     yaml_path = hf_hub_download(repo_id=repo_id, filename="eval.yaml", repo_type="dataset", revision=revision)
 
     with open(yaml_path, "r") as f:
         global_config = yaml.safe_load(f)
 
-    task_config = global_config["tasks"][0]
+    task_configs = global_config["tasks"]
 
-    @task
-    def task_func():
-        return create_task_from_config(repo_id, revision, task_config, global_config)
+    tasks = []
+    for task_config in task_configs:
+        tasks.append(create_task_from_config(repo_id, revision, task_config))
 
-    return task_func
+    return tasks

src/lighteval/main_inspect.py

@@ -522,10 +522,10 @@ def eval(  # noqa C901
         print("run 'inspect view' to view the results")
 
 
-def from_hub(model: str, repo_id: str, limit: int = 100, revision: str = "main"):
+def from_hub(repo_id: str, models: list[str], limit: int = 100, revision: str = "main"):
     task = create_task_function(repo_id, revision)
-    model = "hf-inference-providers/meta-llama/Llama-3.1-8B-Instruct"
-    inspect_ai_eval(tasks=[task], model=model, limit=100)
+
+    inspect_ai_eval(tasks=task, model=models, limit=limit)
 
 
 def bundle(log_dir: str, output_dir: str, overwrite: bool = True, repo_id: str | None = None, public: bool = False):