fix task_list; include_path takes priority

Author: baberabb

lm_eval/tasks/factory.py

@@ -120,6 +120,16 @@ def _load_full_config(
                 "metadata": {"config": "unknown"}
             }  # python task without YAML
 
+        # Handle task_list configs - merge base config with per-task overrides
+        if "task_list" in cfg:
+            task_list = cfg.pop("task_list")
+            # Find the entry for this task in task_list
+            for item in task_list:
+                if isinstance(item, dict) and item.get("task") == entry.name:
+                    # Merge per-task overrides
+                    cfg = {**cfg, **item}
+                    break
+
         if overrides:
             cfg = {**cfg, **overrides}
         cfg["metadata"] = (

lm_eval/tasks/manager.py

@@ -33,8 +33,7 @@ def __init__(
         self._factory = TaskFactory(meta=metadata)
 
         all_paths: list[Path] = []
-        if include_defaults:
-            all_paths.append(Path(__file__).parent)
+        # Process include_path FIRST so user tasks take precedence over defaults
         if include_path:
             all_paths += [
                 Path(p)
@@ -44,6 +43,8 @@ def __init__(
                     else [include_path]
                 )
             ]
+        if include_defaults:
+            all_paths.append(Path(__file__).parent)
 
         self._index = index.build(all_paths)
 

tests/test_configs/group.yaml

@@ -0,0 +1,32 @@
+# Group configuration demonstrating task collections
+
+group: test_group
+task:
+  - task: group_task_fs0
+    dataset_path: json
+    dataset_kwargs:
+      data_files:
+        test: tests/test_configs/test_data.json
+    output_type: multiple_choice
+    doc_to_text: "{{question}}"
+    doc_to_target: "{{choices[answer]}}"
+    test_split: test
+    num_fewshot: 0
+    metric_list:
+      - metric: acc
+        aggregation: mean
+        higher_is_better: true
+  - task: group_task_fs2
+    dataset_path: json
+    dataset_kwargs:
+      data_files:
+        test: tests/test_configs/test_data.json
+    output_type: multiple_choice
+    doc_to_text: "{{question}}"
+    doc_to_target: "{{choices[answer]}}"
+    test_split: test
+    num_fewshot: 2
+    metric_list:
+      - metric: acc
+        aggregation: mean
+        higher_is_better: true

tests/test_configs/include_base.yaml

@@ -0,0 +1,20 @@
+# Base configuration for include walkthrough tests
+# This will be included by other configs to demonstrate inheritance
+
+task: base_task # This should be overridden by including configs
+dataset_path: json
+dataset_kwargs:
+  data_files:
+    test: tests/test_configs/test_data.json
+output_type: multiple_choice
+doc_to_text: "{{question}}"
+doc_to_target: "{{choices[answer]}}"
+test_split: test
+num_fewshot: 0 # Default, can be overridden
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+  description: "Base config for include demonstration"

tests/test_configs/include_group.yaml

@@ -0,0 +1,8 @@
+# Group with multiple tasks using include inheritance
+# Demonstrates tasks sharing the same base config
+
+group: include_group
+task:
+  - include_task_fs0
+  - include_task_fs1
+  - include_task_fs5

tests/test_configs/include_task_fs0.yaml

@@ -0,0 +1,6 @@
+# Task demonstrating include inheritance
+
+task: include_task_fs0
+include: include_base.yaml
+num_fewshot: 0
+description: "Zero-shot with inheritance"

tests/test_configs/include_task_fs1.yaml

@@ -0,0 +1,6 @@
+# Task demonstrating include inheritance
+
+task: include_task_fs1
+include: include_base.yaml
+num_fewshot: 1
+description: "One-shot with inheritance"

tests/test_configs/include_task_fs5.yaml

@@ -0,0 +1,13 @@
+# Task demonstrating include inheritance with custom metrics
+
+task: include_task_fs5
+include: include_base.yaml
+num_fewshot: 5
+description: "Five-shot with custom metrics"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true

tests/test_configs/simple_task.yaml

@@ -0,0 +1,20 @@
+# Simple task configuration for walkthrough tests
+# Demonstrates basic task loading without any special features
+
+task: simple_task
+dataset_path: json
+dataset_kwargs:
+  data_files:
+    test: tests/test_configs/test_data.json
+output_type: multiple_choice
+doc_to_text: "{{question}}"
+doc_to_target: "{{choices[answer]}}"
+test_split: test
+num_fewshot: 1
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+  description: "Simple task for basic walkthrough"

tests/test_configs/task_list.yaml

@@ -0,0 +1,36 @@
+# Task list configuration for code walkthrough tests
+# This demonstrates the task_list feature with shared config and task-specific overrides
+
+dataset_path: json
+dataset_kwargs:
+  data_files:
+    test: tests/test_configs/test_data.json
+output_type: multiple_choice
+doc_to_text: "{{question}}"
+doc_to_target: "{{choices[answer]}}"
+test_split: test
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
+  description: "Task list walkthrough example"
+
+task_list:
+  - task: task_list_fs0
+    num_fewshot: 0
+    description: "Zero-shot variant"
+  - task: task_list_fs1
+    num_fewshot: 1
+    description: "One-shot variant"
+  - task: task_list_fs3
+    num_fewshot: 3
+    description: "Three-shot variant"
+    metric_list:
+      - metric: acc
+        aggregation: mean
+        higher_is_better: true
+      - metric: acc_norm
+        aggregation: mean
+        higher_is_better: true