Skip to content

Commit 08bdaeb

Browse files
committed
fix: restore task list display logic
Updated print_all_tasks to handle the new registry key format introduced in #1038. This fixes the issue where 'tasks list' incorrectly showed 0 tasks.
1 parent 06aee5b commit 08bdaeb

File tree

1 file changed

+7
-76
lines changed

1 file changed

+7
-76
lines changed

src/lighteval/tasks/registry.py

Lines changed: 7 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -88,27 +88,6 @@ def load_community_tasks():
8888
# Custom is for all the experiments you might want to do!
8989

9090
# Core suites - always available without extra dependencies
91-
CORE_SUITES = [
92-
"helm",
93-
"bigbench",
94-
"harness",
95-
"leaderboard",
96-
"lighteval",
97-
"original",
98-
"extended",
99-
"custom",
100-
"test",
101-
]
102-
103-
# Optional suites - may require extra dependencies
104-
OPTIONAL_SUITES = [
105-
"community",
106-
"multilingual",
107-
]
108-
109-
DEFAULT_SUITES = CORE_SUITES + OPTIONAL_SUITES
110-
111-
11291
class Registry:
11392
"""The Registry class is used to manage the task registry and get task classes."""
11493

@@ -379,65 +358,17 @@ def load_all_task_configs(
379358
logger.info(f"Loaded {len(loaded_configs)} task configs in {time_end - time_start:.1f} seconds")
380359
return loaded_configs
381360

382-
def print_all_tasks(self, suites: str | None = None):
383-
"""Print all the tasks in the task registry.
384-
385-
Args:
386-
suites: Comma-separated list of suites to display. If None, shows core suites only.
387-
Use 'all' to show all available suites (core + optional).
388-
Special handling for 'multilingual' suite with dependency checking.
389-
"""
390-
# Parse requested suites
391-
if suites is None:
392-
requested_suites = CORE_SUITES.copy()
393-
else:
394-
requested_suites = [s.strip() for s in suites.split(",")]
395-
396-
# Handle 'all' special case
397-
if "all" in requested_suites:
398-
requested_suites = DEFAULT_SUITES.copy()
399-
400-
# Check for multilingual dependencies if requested
401-
if "multilingual" in requested_suites:
402-
import importlib.util
361+
def print_all_tasks(self):
362+
"""Print all the tasks in the task registry."""
363+
all_tasks = sorted(self._task_registry.keys())
403364

404-
if importlib.util.find_spec("langcodes") is None:
405-
logger.warning(
406-
"Multilingual tasks require additional dependencies (langcodes). "
407-
"Install them with: pip install langcodes"
408-
)
409-
requested_suites.remove("multilingual")
410-
411-
# Get all tasks and filter by requested suites
412-
all_tasks = list(self._task_registry.keys())
413-
tasks_names = [task for task in all_tasks if task.split("|")[0] in requested_suites]
414-
415-
# Ensure all requested suites are present (even if empty)
416-
suites_in_registry = {name.split("|")[0] for name in tasks_names}
417-
for suite in requested_suites:
418-
if suite not in suites_in_registry:
419-
# We add a dummy task to make sure the suite is printed
420-
tasks_names.append(f"{suite}|")
421-
422-
tasks_names.sort()
423-
424-
print(f"Displaying tasks for suites: {', '.join(requested_suites)}")
365+
print(f"Available tasks ({len(all_tasks)}):")
425366
print("=" * 60)
426367

427-
for suite, g in groupby(tasks_names, lambda x: x.split("|")[0]):
428-
tasks_in_suite = [name for name in g if name.split("|")[1]] # Filter out dummy tasks
429-
tasks_in_suite.sort()
430-
431-
print(f"\n- {suite}:")
432-
if not tasks_in_suite:
433-
print(" (no tasks in this suite)")
434-
else:
435-
for task_name in tasks_in_suite:
436-
print(f" - {task_name}")
368+
for task_name in all_tasks:
369+
print(f" - {task_name}")
437370

438-
# Print summary
439-
total_tasks = len([t for t in tasks_names if t.split("|")[1]])
440-
print(f"\nTotal tasks displayed: {total_tasks}")
371+
print(f"\nTotal tasks: {len(all_tasks)}")
441372

442373
def get_tasks_dump(self) -> list[dict]: # noqa: C901
443374
"""Get all task names, metadata, and docstrings as a Python object.

0 commit comments

Comments
 (0)