namescape groups

Author: baberabb

lm_eval/tasks/_config_loader.py

@@ -173,13 +173,14 @@ def load_yaml(
     merged = {}
     for inc in includes if isinstance(includes, list) else [includes]:
         inc_path = (path.parent / inc) if not Path(inc).is_absolute() else Path(inc)
-        merged.update(
-            load_yaml(
-                inc_path,
-                resolve_func=resolve_func,
-                recursive=True,
-                _seen=_seen,
-            ),
+        inc_cfg = load_yaml(
+            inc_path,
+            resolve_func=resolve_func,
+            recursive=True,
+            _seen=_seen,
         )
+        # Don't inherit task_list - it defines tasks for the included file only
+        inc_cfg.pop("task_list", None)
+        merged.update(inc_cfg)
     merged.update(cfg)  # local keys win
     return merged

lm_eval/tasks/arc/arc_challenge.yaml

@@ -1,3 +0,0 @@
-include: arc_easy.yaml
-task: arc_challenge
-dataset_name: ARC-Challenge

lm_eval/tasks/arc/arc_easy.yaml

@@ -21,3 +21,7 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 1.0
+
+task_list:
+  - task: arc_challenge
+    dataset_name: ARC-Challenge

lm_eval/tasks/factory.py

@@ -73,29 +73,45 @@ def _build_group(
         grp_cfg["metadata"] = grp_cfg.get("metadata", {}) | self._meta
         # Use ConfigurableGroup (hashable) instead of GroupConfig (dict, unhashable)
         group_obj = ConfigurableGroup(config=grp_cfg)
+        group_name = entry.name
 
         children: dict[str, Any] = {}
         for item in group_obj.config["task"]:
-            if isinstance(item, str):  # task: hellaswag
+            if isinstance(item, str):
+                # Case 1: String reference - look up in registry
+                base_name = item
                 child = self.build(
                     registry[item],
                     overrides=overrides,  # group-level overrides propagate
                     registry=registry,
                 )
-            elif isinstance(item, dict):  # task: {task: hellaswag, num_fewshot: 5}
+            elif isinstance(item, dict):
                 base_name = item["task"]
-                child = self.build(
-                    registry[base_name],
-                    overrides=item,  # per-item override
-                    registry=registry,
-                )
+                if base_name in registry:
+                    # Case 2: Modify existing indexed task
+                    child = self.build(
+                        registry[base_name],
+                        overrides=item,  # per-item override
+                        registry=registry,
+                    )
+                else:
+                    # Case 3: Create new task inline (not indexed)
+                    task_cfg = {**item}
+                    task_cfg["metadata"] = task_cfg.get("metadata", {}) | self._meta
+                    task_obj = ConfigurableTask(config=task_cfg)
+                    child = {base_name: task_obj}
             else:
                 raise TypeError(
                     f"Unsupported sub-entry {item!r} in group '{entry.name}'"
                 )
 
-            # `child` itself is a mapping (task-name -> obj) or {ConfigurableGroup: ...}
-            children.update(child)
+            # Namespace ALL child tasks with group_name::task_name
+            namespaced_child = {}
+            for task_name, task_obj in child.items():
+                namespaced_name = f"{group_name}::{task_name}"
+                namespaced_child[namespaced_name] = task_obj
+            children.update(namespaced_child)
+
         return {group_obj: children}
 
     def _build_tag(

lm_eval/tasks/index.py

@@ -144,16 +144,45 @@ def process_cfg(
             return
 
         if kind is Kind.TASK_LIST:
+            # If config also has a top-level "task", register it as the base task
+            if "task" in cfg and isinstance(cfg["task"], str):
+                base_name = cfg["task"]
+                if base_name not in index:
+                    index[base_name] = Entry(
+                        name=base_name,
+                        kind=Kind.TASK,
+                        yaml_path=path,
+                        tags=TaskIndex._str_to_set(cfg.get("tag")),
+                        cfg=cfg,
+                    )
+                    TaskIndex._register_tags(base_name, cfg.get("tag"), index)
+
+            # Register each task in task_list
+            base_tag = cfg.get("tag")
             for entry in cfg["task_list"]:
                 task_name = entry["task"] if isinstance(entry, dict) else entry
+                if task_name in index:
+                    log.warning(
+                        f"Duplicate task name '{task_name}' found. "
+                        f"Already registered from: {index[task_name].yaml_path}. "
+                        f"Skipping duplicate from: {path}"
+                    )
+                    continue
+                # Combine base tag with per-entry tag
+                entry_tag = entry.get("tag") if isinstance(entry, dict) else None
+                combined_tags = TaskIndex._str_to_set(base_tag) | TaskIndex._str_to_set(
+                    entry_tag
+                )
                 index[task_name] = Entry(
                     name=task_name,
                     kind=Kind.TASK,
                     yaml_path=path,
-                    tags=TaskIndex._str_to_set(cfg.get("tag")),
+                    tags=combined_tags,
                     cfg=cfg,
                 )
-                TaskIndex._register_tags(task_name, entry.get("tag"), index)
+                # Register both base config's tag and per-entry tag
+                TaskIndex._register_tags(task_name, base_tag, index)
+                TaskIndex._register_tags(task_name, entry_tag, index)
             return
 
     @staticmethod

tests/test_task_manager.py

@@ -375,6 +375,13 @@ def shared_task_manager():
     return TaskManager()
 
 
+@pytest.fixture(scope="module")
+def test_configs_task_manager():
+    """TaskManager with only test_configs tasks (fast - no default task scanning)"""
+    test_configs_path = Path(__file__).parent / "test_configs"
+    return TaskManager(include_path=str(test_configs_path), include_defaults=False)
+
+
 class TestTaskManagerIntegration:
     def test_initialization(self, shared_task_manager):
         """TaskManager initializes with default tasks"""
@@ -407,16 +414,20 @@ def test_all_tags_property(self, shared_task_manager):
         for t in tags[:5]:  # Check first 5
             assert shared_task_manager._name_is_tag(t)
 
-    def test_load_task_by_name(self, shared_task_manager):
+    def test_load_task_by_name(self, test_configs_task_manager):
         """Load a single task by name"""
-        result = shared_task_manager.load_task_or_group(["arc_easy"])
-        assert "arc_easy" in result
-
-    def test_load_group_by_name(self, shared_task_manager):
-        """Load a group and get nested structure"""
-        result = shared_task_manager.load_task_or_group(["ai2_arc"])
-        # ai2_arc is a tag that contains arc_easy and arc_challenge
-        assert "arc_easy" in result or "arc_challenge" in result
+        result = test_configs_task_manager.load_task_or_group(["simple_task"])
+        assert "simple_task" in result
+
+    def test_load_group_by_name(self, test_configs_task_manager):
+        """Load a group and get nested structure with namespaced task names"""
+        result = test_configs_task_manager.load_task_or_group(["test_group"])
+        # Result is {ConfigurableGroup: {task_name: task_obj}}
+        # Get the children dict from the group
+        children = list(result.values())[0]
+        # test_group contains inline tasks, namespaced as group_name::task_name
+        assert "test_group::group_task_fs0" in children
+        assert "test_group::group_task_fs2" in children
 
     def test_load_tag_by_name(self, shared_task_manager):
         """Load all tasks in a tag"""
@@ -522,6 +533,33 @@ def test_task_list_overrides(self):
         assert "acc" in metric_names
         assert "acc_norm" in metric_names
 
+    def test_task_list_base_field_inheritance(self):
+        """Test that task_list tasks inherit base fields from the shared config"""
+        test_configs_path = Path(__file__).parent / "test_configs"
+        tm = TaskManager(include_path=str(test_configs_path), include_defaults=False)
+
+        result = tm.load_task_or_group(["task_list_fs0"])
+        task = result["task_list_fs0"]
+
+        # Base fields should be inherited from the shared config
+        assert task.config.dataset_path == "json", (
+            "Should inherit dataset_path from base"
+        )
+        assert task.config.output_type == "multiple_choice", (
+            "Should inherit output_type from base"
+        )
+        assert task.config.doc_to_text == "{{question}}", (
+            "Should inherit doc_to_text from base"
+        )
+        assert task.config.test_split == "test", "Should inherit test_split from base"
+
+        # Default metric_list should be inherited (task_list_fs0 doesn't override it)
+        metric_names = [m["metric"] for m in task.config.metric_list]
+        assert "acc" in metric_names, "Should inherit metric_list from base"
+
+        # Per-task override should still be applied
+        assert task.config.num_fewshot == 0, "Should have per-task num_fewshot override"
+
     def test_match_tasks_glob(self, shared_task_manager):
         """match_tasks handles glob patterns"""
         matches = shared_task_manager.match_tasks(["arc_*"])
@@ -543,7 +581,7 @@ def test_name_is_tag(self, shared_task_manager):
         assert shared_task_manager._name_is_tag("ai2_arc")
         assert not shared_task_manager._name_is_tag("arc_easy")  # This is a task
 
-    def test_include_path_precedence(self):
+    def test_include_path_precedence(self, shared_task_manager):
         """Test that user-specified include paths take precedence over default paths when tasks have the same name."""
         with tempfile.TemporaryDirectory() as custom_dir:
             # Create a custom arc_easy.yaml that has a different metric
@@ -589,8 +627,8 @@ def test_include_path_precedence(self):
             )
 
             # Test 2: Verify default is used when no custom path is provided
-            default_task_manager = TaskManager(include_defaults=True)
-            default_task_dict = default_task_manager.load_task_or_group(["arc_easy"])
+            # Use shared_task_manager instead of creating a new one (saves ~9s)
+            default_task_dict = shared_task_manager.load_task_or_group(["arc_easy"])
             default_arc_easy = default_task_dict["arc_easy"]
 
             # Default should not have f1 metric or custom text