|
1 | 1 | """
|
2 |
| -NOTE: models produced using this example will not be capable of running in vLLM. |
3 |
| -You will also need to install `transformers>=4.56` or install from source |
4 |
| -See https://github.com/vllm-project/vllm/pull/22219 for progress updates |
| 2 | +WARNING: This example requires the following minimum versions: |
| 3 | + * compressed-tensors>=0.10.3.dev |
| 4 | + * transformers>=4.56.dev |
| 5 | +Note that (you may need to install from source) |
| 6 | +
|
| 7 | +Models produced by this example will not be runnable in vLLM without |
| 8 | +the following changes: https://github.com/vllm-project/vllm/pull/22219 |
5 | 9 | """
|
6 | 10 |
|
7 | 11 | from datasets import load_dataset
|
| 12 | +from packaging import version |
8 | 13 | from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 14 | +from transformers.utils.import_utils import _is_package_available |
9 | 15 |
|
10 | 16 | from llmcompressor import oneshot
|
11 | 17 | from llmcompressor.modifiers.quantization import QuantizationModifier
|
12 | 18 | from llmcompressor.modifiers.transform import QuIPModifier
|
13 | 19 | from llmcompressor.utils import dispatch_for_generation
|
14 | 20 |
|
| 21 | +# check correct versioning |
| 22 | +_, ct_version = _is_package_available("compressed_tensors", return_version=True) |
| 23 | +_, tfms_version = _is_package_available("transformers", return_version=True) |
| 24 | +if version.parse(ct_version) < version.parse("0.10.3.dev"): |
| 25 | + print(version.parse(ct_version)) |
| 26 | + raise ValueError("Please install compressed-tensors>=0.10.3 or from source") |
| 27 | +if version.parse(tfms_version) < version.parse("4.56.dev"): |
| 28 | + raise ValueError("Please install transformers>=4.56 or from source") |
| 29 | + |
15 | 30 | # Select model and load it.
|
16 | 31 | MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct"
|
17 | 32 |
|
|
0 commit comments