|
1 | 1 | # MIT License |
2 | | - |
| 2 | +# |
3 | 3 | # Copyright (c) 2024 The HuggingFace Team |
4 | | - |
| 4 | +# |
5 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy |
6 | 6 | # of this software and associated documentation files (the "Software"), to deal |
7 | 7 | # in the Software without restriction, including without limitation the rights |
8 | 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
9 | 9 | # copies of the Software, and to permit persons to whom the Software is |
10 | 10 | # furnished to do so, subject to the following conditions: |
11 | | - |
| 11 | +# |
12 | 12 | # The above copyright notice and this permission notice shall be included in all |
13 | 13 | # copies or substantial portions of the Software. |
14 | | - |
| 14 | +# |
15 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
20 | 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
21 | 21 | # SOFTWARE. |
22 | 22 |
|
23 | | -# tests/utils/pretend_missing.py |
24 | | -import functools |
25 | | -import importlib |
26 | | - |
27 | 23 | import pytest |
28 | | -from packaging.requirements import Requirement |
29 | | - |
30 | | -import lighteval.utils.imports as imports |
31 | | - |
32 | | - |
33 | | -def pretend_missing(*names): |
34 | | - """ |
35 | | - Decorator: pretend that certain packages are missing |
36 | | - by patching mypkg.utils.is_package_available. |
37 | | - """ |
38 | | - |
39 | | - def decorator(test_func): |
40 | | - @functools.wraps(test_func) |
41 | | - def wrapper(*args, **kwargs): |
42 | | - from unittest.mock import patch |
43 | | - |
44 | | - def fake(requirement): |
45 | | - name = requirement.name if isinstance(requirement, Requirement) else requirement |
46 | | - return False if name in names else (importlib.util.find_spec(name) is not None) |
47 | 24 |
|
48 | | - with patch.object(imports, "is_package_available", side_effect=fake): |
49 | | - # If your module caches results at import time, reload here |
50 | | - import lighteval |
| 25 | +from lighteval.utils.imports import Extra, is_package_available, requires |
51 | 26 |
|
52 | | - importlib.reload(lighteval) |
53 | 27 |
|
54 | | - return test_func(*args, **kwargs) |
| 28 | +def test_requires(): |
| 29 | + @requires("sglang") |
| 30 | + class RandomModel: |
| 31 | + pass |
55 | 32 |
|
56 | | - return wrapper |
| 33 | + assert RandomModel.__name__ == "RandomModel" |
| 34 | + assert RandomModel.__class__.__name__ == "DummyObject" |
57 | 35 |
|
58 | | - return decorator |
| 36 | + with pytest.raises( |
| 37 | + ImportError, |
| 38 | + match="Through the use of RandomModel, you requested the use of `sglang` for this evaluation, but it is not available in your current environment. Please install it using pip.", |
| 39 | + ): |
| 40 | + RandomModel() |
59 | 41 |
|
60 | 42 |
|
61 | | -@pretend_missing("langdetect") |
62 | | -def test_langdetect_required_for_ifeval(): |
63 | | - from lighteval.main_accelerate import accelerate |
| 43 | +def test_requires_with_extra(): |
| 44 | + @requires(Extra.TGI) |
| 45 | + class RandomModel: |
| 46 | + pass |
64 | 47 |
|
65 | 48 | with pytest.raises( |
66 | 49 | ImportError, |
67 | | - match="Through the use of ifeval_prompt, you requested the use of `langdetect` for this evaluation, but it is not available in your current environment. Please install it using pip.", |
| 50 | + match=r"Through the use of RandomModel, you are trying to run an evaluation requiring tgi capabilities. Please install the required extra: `pip install lighteval\[tgi\]`", |
68 | 51 | ): |
69 | | - accelerate(model_args="model_name=gpt2,batch_size=1", tasks="extended|ifeval|0", max_samples=0) |
| 52 | + RandomModel() |
70 | 53 |
|
71 | 54 |
|
72 | | -@pretend_missing("spacy", "stanza") |
73 | | -def test_multilingual_required_for_xnli(): |
74 | | - """ |
75 | | - This checks that the Extra.MULTILINGUAL correctly raises if there are missing dependencies. |
76 | | - """ |
77 | | - from lighteval.main_accelerate import accelerate |
78 | | - |
| 55 | +def test_requires_with_wrong_dependency(): |
79 | 56 | with pytest.raises( |
80 | | - ImportError, |
81 | | - match="Through the use of get_multilingual_normalizer, you are trying to run an evaluation requiring multilingual capabilities.", |
| 57 | + RuntimeError, |
| 58 | + match="A dependency was specified with @requires, but it is not defined in the possible dependencies defined in the pyproject.toml: `random_dependency`", |
82 | 59 | ): |
83 | | - accelerate(model_args="model_name=gpt2,batch_size=1", tasks="multilingual|xnli_zho_mcf|0", max_samples=0) |
84 | 60 |
|
| 61 | + @requires("random_dependency") |
| 62 | + class RandomModel: |
| 63 | + pass |
85 | 64 |
|
86 | | -@pretend_missing("vllm") |
87 | | -def test_vllm_required_for_vllm_usage(): |
88 | | - from lighteval.main_vllm import vllm |
89 | 65 |
|
| 66 | +def test_is_package_available(): |
| 67 | + assert is_package_available("torch") |
| 68 | + |
| 69 | + |
| 70 | +def test_is_package_unavailable(): |
| 71 | + assert not is_package_available("tensorboardX") |
| 72 | + |
| 73 | + |
| 74 | +def test_is_package_is_not_specified_in_pyproject_toml(): |
90 | 75 | with pytest.raises( |
91 | | - ImportError, |
92 | | - match="Through the use of VLLMModel, you requested the use of `vllm<0.10.2,>=0.10.0` for this evaluation, but it is not available in your current environment. Please install it using pip.", |
| 76 | + RuntimeError, |
| 77 | + match="Package tensorflow was tested against, but isn't specified in the pyproject.toml file. Please specifyit as a potential dependency or an extra for it to be checked.", |
93 | 78 | ): |
94 | | - vllm(model_args="model_name=gpt2", tasks="lighteval|aime24|0", max_samples=0) |
| 79 | + is_package_available("tensorflow") |
0 commit comments