Tests!

Author: LysandreJik

src/lighteval/metrics/imports/data_stats_metric.py

@@ -30,7 +30,7 @@
 from typing import Literal
 
 from lighteval.metrics.imports.data_stats_utils import Fragments
-from lighteval.utils.imports import raise_if_package_not_available
+from lighteval.utils.imports import Extras, raise_if_package_not_available, requires
 
 
 logger = logging.getLogger(__name__)
@@ -55,6 +55,7 @@ def find_ngrams(input_list, n):
     return zip(*[input_list[i:] for i in range(n)])
 
 
+@requires(Extras.MULTILINGUAL)
 class DataStatsMetric(Metric):
     def __init__(
         self,

src/lighteval/metrics/normalizations.py

@@ -28,6 +28,7 @@
 from typing import Callable
 
 from lighteval.metrics.utils.linguistic_tokenizers import get_word_tokenizer
+from lighteval.utils.imports import Extras, requires
 from lighteval.utils.language import Language
 
 
@@ -444,6 +445,7 @@ def remove_punc(text: str) -> str:
     return "".join(ch for ch in text if ch not in PUNCT)
 
 
+@requires(Extras.MULTILINGUAL)
 def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]:
     """Get a normalizer function for the specified language.
 

src/lighteval/metrics/utils/linguistic_tokenizers.py

@@ -20,6 +20,7 @@
 from lighteval.utils.imports import (
     Extras,
     raise_if_package_not_available,
+    requires,
 )
 from lighteval.utils.language import Language
 
@@ -97,11 +98,10 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
         return list(self.tokenizer.span_tokenize(text))
 
 
+@requires(Extras.MULTILINGUAL)
 class SpaCyTokenizer(WordTokenizer):
     def __init__(self, spacy_language: str, config=None):
         super().__init__()
-        raise_if_package_not_available(Extras.MULTILINGUAL, language=spacy_language)
-
         self.spacy_language = spacy_language
         self.config = config
         self._tokenizer = None
@@ -135,6 +135,7 @@ def span_tokenize(self, text: str) -> list[tuple[int, int]]:
         ]
 
 
+@requires("stanza")
 class StanzaTokenizer(WordTokenizer):
     def __init__(self, stanza_language: str, **stanza_kwargs):
         super().__init__()

src/lighteval/tasks/extended/ifbench/instructions.py

@@ -25,8 +25,15 @@
 
 import emoji
 import nltk
-import spacy
-import syllapy
+
+from lighteval.utils.imports import is_package_available, requires
+
+
+if is_package_available("syllapy"):
+    import syllapy
+
+if is_package_available("spacy"):
+    import spacy
 
 import lighteval.tasks.extended.ifeval.instructions_utils as instructions_util
 
@@ -61,6 +68,7 @@
 RESOURCES_DOWNLOADED: bool = False
 
 
+@requires("syllapy", "spacy")
 class Instruction:
     """An instruction template."""
 

src/lighteval/utils/imports.py

@@ -53,28 +53,30 @@ def is_multilingual_package_available(language: str):
 def raise_if_package_not_available(package_name: str | Extras, *, language: str = None, object_name: str = None):
     prefix = "You" if object_name is None else f"Through the use of {object_name}, you"
 
-    if package_name == Extras.MULTILINGUAL and not is_multilingual_package_available(language):
+    if package_name == Extras.MULTILINGUAL and (
+        (language is not None) or (not is_multilingual_package_available(language))
+    ):
         raise ImportError(prefix + not_installed_error_message(package_name)[3:])
 
     if not is_package_available(package_name):
         raise ImportError(prefix + not_installed_error_message(package_name)[3:])
 
 
 def not_installed_error_message(package_name: str | Extras) -> str:
-    if package_name == Extras.MULTILINGUAL:
+    if package_name == Extras.MULTILINGUAL.value:
         return "You are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`"
-    elif package_name == Extras.EXTENDED:
+    elif package_name == Extras.EXTENDED.value:
         return "You are trying to run an evaluation requiring additional extensions. Please install the required extra: `pip install lighteval[extended] "
     elif package_name == "text_generation":
-        return "You are trying to start a text generation inference endpoint, but TGI is not present in your local environement. Please install it using pip."
+        return "You are trying to start a text generation inference endpoint, but TGI is not present in your local environment. Please install it using pip."
     elif package_name in ["bitsandbytes", "auto-gptq"]:
-        return f"You are trying to load a model quantized with `{package_name}`, which is not available in your local environement. Please install it using pip."
+        return f"You are trying to load a model quantized with `{package_name}`, which is not available in your local environment. Please install it using pip."
     elif package_name == "peft":
         return "You are trying to use adapter weights models, for which you need `peft`, which is not available in your environment. Please install it using pip."
     elif package_name == "openai":
         return "You are trying to use an Open AI LLM as a judge, for which you need `openai`, which is not available in your environment. Please install it using pip."
 
-    return f"You requested the use of `{package_name}` for this evaluation, but it is not available in your current environement. Please install it using pip."
+    return f"You requested the use of `{package_name}` for this evaluation, but it is not available in your current environment. Please install it using pip."
 
 
 class DummyObject(type):

tests/test_dependencies.py

@@ -0,0 +1,89 @@
+# MIT License
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# tests/utils/pretend_missing.py
+import functools
+import importlib
+
+import pytest
+
+import lighteval.utils.imports as imports
+
+
+def pretend_missing(*names):
+    """
+    Decorator: pretend that certain packages are missing
+    by patching mypkg.utils.is_package_available.
+    """
+
+    def decorator(test_func):
+        @functools.wraps(test_func)
+        def wrapper(*args, **kwargs):
+            from unittest.mock import patch
+
+            def fake(name):
+                return False if name in names else (importlib.util.find_spec(name) is not None)
+
+            with patch.object(imports, "is_package_available", side_effect=fake):
+                # If your module caches results at import time, reload here
+                import lighteval
+
+                importlib.reload(lighteval)
+
+                return test_func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@pretend_missing("langdetect")
+def test_langdetect_required_for_ifeval():
+    from lighteval.main_accelerate import accelerate
+
+    with pytest.raises(
+        ImportError,
+        match="Through the use of ifeval_prompt, you requested the use of `langdetect` for this evaluation, but it is not available in your current environment. Please install it using pip.",
+    ):
+        accelerate(model_args="model_name=gpt2,batch_size=1", tasks="extended|ifeval|0", max_samples=0)
+
+
+@pretend_missing("spacy", "stanza")
+def test_multilingual_required_for_xnli():
+    from lighteval.main_accelerate import accelerate
+
+    with pytest.raises(
+        ImportError,
+        match="Through the use of get_multilingual_normalizer, you are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`",
+    ):
+        accelerate(model_args="model_name=gpt2,batch_size=1", tasks="lighteval|xnli_zho_mcf|0", max_samples=0)
+
+
+@pretend_missing("vllm")
+def test_vllm_required_for_vllm_usage():
+    from lighteval.main_vllm import vllm
+
+    with pytest.raises(
+        ImportError,
+        match="You requested the use of `vllm` for this evaluation, but it is not available in your current environment. Please install it using pip.'",
+    ):
+        vllm(model_args="model_name=gpt2,batch_size=1", tasks="lighteval|xnli_zho_mcf|0", max_samples=0)