Update

Author: LysandreJik

pyproject.toml

@@ -84,7 +84,8 @@ dependencies = [
     "fsspec>=2023.12.2",
     "httpx>=0.27.2",
     "latex2sympy2_extended==1.0.6",
-    "langcodes"
+    "langcodes",
+    "sglang"
 ]
 
 [project.optional-dependencies]

src/lighteval/metrics/imports/data_stats_metric.py

@@ -30,7 +30,7 @@
 from typing import Literal
 
 from lighteval.metrics.imports.data_stats_utils import Fragments
-from lighteval.utils.imports import Extras, raise_if_package_not_available, requires
+from lighteval.utils.imports import Extra, requires
 
 
 logger = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def find_ngrams(input_list, n):
     return zip(*[input_list[i:] for i in range(n)])
 
 
-@requires(Extras.MULTILINGUAL)
+@requires(Extra.MULTILINGUAL)
 class DataStatsMetric(Metric):
     def __init__(
         self,
@@ -87,7 +87,6 @@ def __init__(
                 determines the spaCy model used for tokenization. Currently supports English,
                 German, French, and Italian.
         """
-        raise_if_package_not_available("spacy")
         import spacy
 
         self.n_gram = n_gram

src/lighteval/metrics/normalizations.py

@@ -28,7 +28,7 @@
 from typing import Callable
 
 from lighteval.metrics.utils.linguistic_tokenizers import get_word_tokenizer
-from lighteval.utils.imports import Extras, requires
+from lighteval.utils.imports import Extra, requires
 from lighteval.utils.language import Language
 
 
@@ -445,7 +445,7 @@ def remove_punc(text: str) -> str:
     return "".join(ch for ch in text if ch not in PUNCT)
 
 
-@requires(Extras.MULTILINGUAL)
+@requires(Extra.MULTILINGUAL)
 def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]:
     """Get a normalizer function for the specified language.
 

src/lighteval/metrics/utils/linguistic_tokenizers.py

@@ -18,8 +18,7 @@
 from typing import Callable, Iterator
 
 from lighteval.utils.imports import (
-    Extras,
-    raise_if_package_not_available,
+    Extra,
     requires,
 )
 from lighteval.utils.language import Language
@@ -98,7 +97,7 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
         return list(self.tokenizer.span_tokenize(text))
 
 
-@requires(Extras.MULTILINGUAL)
+@requires(Extra.MULTILINGUAL)
 class SpaCyTokenizer(WordTokenizer):
     def __init__(self, spacy_language: str, config=None):
         super().__init__()
@@ -139,7 +138,6 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
 class StanzaTokenizer(WordTokenizer):
     def __init__(self, stanza_language: str, **stanza_kwargs):
         super().__init__()
-        raise_if_package_not_available("stanza")
         self.stanza_language = stanza_language
         self.stanza_kwargs = stanza_kwargs
         self._tokenizer = None

src/lighteval/models/endpoints/tgi_model.py

@@ -32,7 +32,7 @@
 from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel
 from lighteval.tasks.prompt_manager import PromptManager
 from lighteval.utils.cache_management import SampleCache
-from lighteval.utils.imports import is_package_available, requires
+from lighteval.utils.imports import Extra, is_package_available, requires
 
 
 if is_package_available("tgi"):
@@ -99,7 +99,7 @@ class TGIModelConfig(ModelConfig):
 
 # inherit from InferenceEndpointModel instead of LightevalModel since they both use the same interface, and only overwrite
 # the client functions, since they use a different client.
-@requires("tgi")
+@requires(Extra.TGI)
 class ModelClient(InferenceEndpointModel):
     _DEFAULT_MAX_LENGTH: int = 4096
 

src/lighteval/models/model_loader.py

@@ -43,7 +43,6 @@
 from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig
 from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModel, VLMTransformersModelConfig
 from lighteval.models.vllm.vllm_model import AsyncVLLMModel, VLLMModel, VLLMModelConfig
-from lighteval.utils.imports import raise_if_package_not_available
 
 
 logger = logging.getLogger(__name__)
@@ -148,7 +147,6 @@ def load_model_with_accelerate_or_default(
     elif isinstance(config, DeltaModelConfig):
         model = DeltaModel(config=config)
     elif isinstance(config, VLLMModelConfig):
-        raise_if_package_not_available("vllm")
         if config.is_async:
             model = AsyncVLLMModel(config=config)
         else:

src/lighteval/pipeline.py

@@ -42,7 +42,7 @@
 from lighteval.tasks.lighteval_task import LightevalTask
 from lighteval.tasks.registry import Registry
 from lighteval.tasks.requests import SamplingMethod
-from lighteval.utils.imports import is_package_available, raise_if_package_not_available
+from lighteval.utils.imports import is_package_available
 from lighteval.utils.parallelism import test_all_gather
 from lighteval.utils.utils import make_results_table, remove_reasoning_tags
 
@@ -96,21 +96,6 @@ class PipelineParameters:
     bootstrap_iters: int = 1000
 
     def __post_init__(self):  # noqa C901
-        # Import testing
-        if self.launcher_type == ParallelismManager.ACCELERATE:
-            raise_if_package_not_available("accelerate")
-        elif self.launcher_type == ParallelismManager.VLLM:
-            raise_if_package_not_available("vllm")
-        elif self.launcher_type == ParallelismManager.SGLANG:
-            raise_if_package_not_available("sglang")
-        elif self.launcher_type == ParallelismManager.TGI:
-            raise_if_package_not_available("tgi")
-        elif self.launcher_type == ParallelismManager.NANOTRON:
-            raise_if_package_not_available("nanotron")
-        elif self.launcher_type == ParallelismManager.OPENAI:
-            raise_if_package_not_available("openai")
-
-        # Convert reasoning tags to list if needed
         if not isinstance(self.reasoning_tags, list):
             try:
                 self.reasoning_tags = ast.literal_eval(self.reasoning_tags)

src/lighteval/tasks/registry.py

@@ -115,9 +115,9 @@ def __init__(
         self,
         tasks: str | Path | None = None,
         custom_tasks: str | Path | ModuleType | None = None,
-        load_community: bool = True,
-        load_extended: bool = True,
-        load_multilingual: bool = True,
+        load_community: bool = False,
+        load_extended: bool = False,
+        load_multilingual: bool = False,
     ):
         """
         Initialize the Registry class.
@@ -213,6 +213,13 @@ def _activate_loading_of_optional_suite(self) -> None:
                     f"Suite {suite_name} unknown. This is not normal, unless you are testing adding new evaluations."
                 )
 
+        if "extended" in suites:
+            self._load_extended = True
+        if "multilingual" in suites:
+            self._load_multilingual = True
+        if "community" in suites:
+            self._load_community = True
+
     def _load_full_registry(self) -> dict[str, LightevalTaskConfig]:
         """
         Returns:

src/lighteval/utils/imports.py

@@ -15,24 +15,64 @@
 import functools
 import importlib
 import inspect
+import re
+from collections import defaultdict
 from functools import lru_cache
+from importlib.metadata import PackageNotFoundError, metadata, version
+from typing import Dict, List, Tuple
 
 from packaging.requirements import Requirement
+from packaging.version import Version
 
 
-class Extras(enum.Enum):
+# These extras should exist in the pyproject.toml file
+class Extra(enum.Enum):
     MULTILINGUAL = "multilingual"
     EXTENDED = "extended"
+    TGI = "tgi"
 
 
 @lru_cache()
-def is_package_available(package_name: str | Extras):
-    if package_name == Extras.MULTILINGUAL:
-        return all(importlib.util.find_spec(package) is not None for package in ["stanza", "spacy"])
-    if package_name == Extras.EXTENDED:
-        return all(importlib.util.find_spec(package) is not None for package in ["spacy"])
+def is_package_available(package: str | Requirement | Extra):
+    deps, deps_by_extra = required_dependencies()
+
+    if isinstance(package, str):
+        package = deps[package]
+
+    if isinstance(package, Extra):
+        dependencies = deps_by_extra[package.value]
+        return all(is_package_available(_package) for _package in dependencies)
     else:
-        return importlib.util.find_spec(package_name) is not None
+        try:
+            installed = Version(version(package.name))
+        except PackageNotFoundError:
+            return False
+
+        # No version constraint → any installed version is OK
+        if not package.specifier:
+            return True
+
+        return installed in package.specifier
+
+
+@lru_cache()
+def required_dependencies() -> Tuple[Dict[str, Requirement], Dict[str, List[Requirement]]]:
+    md = metadata("lighteval")
+    requires_dist = md.get_all("Requires-Dist") or []
+    deps_by_extra = defaultdict(list)
+    deps = {}
+
+    for dep in requires_dist:
+        extra = None
+        if ";" in dep:
+            dep, marker = dep.split(";", 1)
+            match = re.search(r'extra\s*==\s*"(.*?)"', marker)
+            extra = match.group(1) if match else None
+        requirement = Requirement(dep.strip())
+        deps_by_extra[extra].append(requirement)
+        deps[requirement.name] = requirement
+
+    return deps, deps_by_extra
 
 
 @lru_cache()
@@ -50,33 +90,32 @@ def is_multilingual_package_available(language: str):
     return all(cur_import is not None for cur_import in imports)
 
 
-def raise_if_package_not_available(package_name: str | Extras, *, language: str = None, object_name: str = None):
+def raise_if_package_not_available(package: Requirement | Extra, *, language: str = None, object_name: str = None):
     prefix = "You" if object_name is None else f"Through the use of {object_name}, you"
 
-    if package_name == Extras.MULTILINGUAL and (
-        (language is not None) or (not is_multilingual_package_available(language))
-    ):
-        raise ImportError(prefix + not_installed_error_message(package_name)[3:])
+    if package == Extra.MULTILINGUAL and ((language is not None) or (not is_multilingual_package_available(language))):
+        raise ImportError(prefix + not_installed_error_message(package)[3:])
 
-    if not is_package_available(package_name):
-        raise ImportError(prefix + not_installed_error_message(package_name)[3:])
+    if not is_package_available(package):
+        raise ImportError(prefix + not_installed_error_message(package)[3:])
 
 
-def not_installed_error_message(package_name: str | Extras) -> str:
-    if package_name == Extras.MULTILINGUAL.value:
+def not_installed_error_message(package: Requirement) -> str:
+    if package == Extra.MULTILINGUAL.value:
         return "You are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`"
-    elif package_name == Extras.EXTENDED.value:
+    elif package == Extra.EXTENDED.value:
         return "You are trying to run an evaluation requiring additional extensions. Please install the required extra: `pip install lighteval[extended] "
-    elif package_name == "text_generation":
+    elif package == "text_generation":
         return "You are trying to start a text generation inference endpoint, but TGI is not present in your local environment. Please install it using pip."
-    elif package_name in ["bitsandbytes", "auto-gptq"]:
-        return f"You are trying to load a model quantized with `{package_name}`, which is not available in your local environment. Please install it using pip."
-    elif package_name == "peft":
+    elif package == "peft":
         return "You are trying to use adapter weights models, for which you need `peft`, which is not available in your environment. Please install it using pip."
-    elif package_name == "openai":
+    elif package == "openai":
         return "You are trying to use an Open AI LLM as a judge, for which you need `openai`, which is not available in your environment. Please install it using pip."
 
-    return f"You requested the use of `{package_name}` for this evaluation, but it is not available in your current environment. Please install it using pip."
+    if isinstance(package, Extra):
+        return f"You are trying to run an evaluation requiring {package.value} capabilities. Please install the required extra: `pip install lighteval[{package.value}]`"
+    else:
+        return f"You requested the use of `{package}` for this evaluation, but it is not available in your current environment. Please install it using pip."
 
 
 class DummyObject(type):
@@ -101,9 +140,22 @@ def requires(*backends):
     which is not installed.
     """
 
+    requirements, _ = required_dependencies()
+
     applied_backends = []
     for backend in backends:
-        applied_backends.append(Requirement(backend.value if isinstance(backend, Extras) else backend))
+        if isinstance(backend, Extra):
+            applied_backends.append(backend)
+        else:
+            if backend not in requirements:
+                raise RuntimeError(
+                    "A dependency was specified with @requires, but it is not defined in the possible dependencies "
+                    f"defined in the pyproject.toml: `{backend}`."
+                    f""
+                    f"If editing the pyproject.toml to add a new dependency, remember to reinstall lighteval for the"
+                    f"update to take effect."
+                )
+            applied_backends.append(requirements[backend])
 
     def inner_fn(_object):
         _object._backends = applied_backends
@@ -115,7 +167,7 @@ class Placeholder(metaclass=DummyObject):
 
                 def __init__(self, *args, **kwargs):
                     for backend in self._backends:
-                        raise_if_package_not_available(backend.name, object_name=_object.__class__.__name__)
+                        raise_if_package_not_available(backend, object_name=_object.__name__)
 
             Placeholder.__name__ = _object.__name__
             Placeholder.__module__ = _object.__module__
@@ -126,7 +178,7 @@ def __init__(self, *args, **kwargs):
             @functools.wraps(_object)
             def wrapper(*args, **kwargs):
                 for backend in _object._backends:
-                    raise_if_package_not_available(backend.name, object_name=_object.__name__)
+                    raise_if_package_not_available(backend, object_name=_object.__name__)
                 return _object(*args, **kwargs)
 
             return wrapper

tests/test_dependencies.py

@@ -25,6 +25,7 @@
 import importlib
 
 import pytest
+from packaging.requirements import Requirement
 
 import lighteval.utils.imports as imports
 
@@ -40,7 +41,8 @@ def decorator(test_func):
         def wrapper(*args, **kwargs):
             from unittest.mock import patch
 
-            def fake(name):
+            def fake(requirement):
+                name = requirement.name if isinstance(requirement, Requirement) else requirement
                 return False if name in names else (importlib.util.find_spec(name) is not None)
 
             with patch.object(imports, "is_package_available", side_effect=fake):
@@ -73,9 +75,9 @@ def test_multilingual_required_for_xnli():
 
     with pytest.raises(
         ImportError,
-        match="Through the use of get_multilingual_normalizer, you are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`",
+        match="Through the use of get_multilingual_normalizer, you are trying to run an evaluation requiring multilingual capabilities.",
     ):
-        accelerate(model_args="model_name=gpt2,batch_size=1", tasks="lighteval|xnli_zho_mcf|0", max_samples=0)
+        accelerate(model_args="model_name=gpt2,batch_size=1", tasks="multilingual|xnli_zho_mcf|0", max_samples=0)
 
 
 @pretend_missing("vllm")
@@ -84,6 +86,6 @@ def test_vllm_required_for_vllm_usage():
 
     with pytest.raises(
         ImportError,
-        match="You requested the use of `vllm` for this evaluation, but it is not available in your current environment. Please install it using pip.'",
+        match="Through the use of VLLMModel, you requested the use of `vllm<0.10.2,>=0.10.0` for this evaluation, but it is not available in your current environment. Please install it using pip.",
     ):
-        vllm(model_args="model_name=gpt2,batch_size=1", tasks="lighteval|xnli_zho_mcf|0", max_samples=0)
+        vllm(model_args="model_name=gpt2", tasks="lighteval|aime24|0", max_samples=0)