Skip to content

Commit 1867c25

Browse files
authored
Fix target matching for fused layers with compressed-tensors (#12617)
Without this PR --------------- Quantizing models with llm-compressor and a recipe that explicitly lists names of layers produces a model that is not loadable by vLLM (i.e. `vllm serve <model>` fails with `raise ValueError(f"Unable to find matching target for {module} in the ...`). Example recipe: ``` recipe = """ quantization_stage: run_type: oneshot quantization_modifiers: GPTQModifier: ignore: ["lm_head"] config_groups: group_0: weights: num_bits: 4 type: "int" symmetric: true strategy: "group" group_size: 128 targets: [ "model.layers.0.mlp.down_proj", "model.layers.2.mlp.down_proj", "model.layers.3.mlp.down_proj", "model.layers.4.mlp.down_proj", "model.layers.5.mlp.down_proj", "model.layers.6.mlp.down_proj", "model.layers.7.mlp.down_proj", "model.layers.8.mlp.down_proj", "model.layers.9.mlp.down_proj", "model.layers.10.mlp.down_proj", "model.layers.11.mlp.down_proj", "model.layers.12.mlp.down_proj", "model.layers.13.mlp.down_proj", "model.layers.14.mlp.down_proj", "model.layers.15.mlp.down_proj", "model.layers.16.mlp.down_proj", "model.layers.17.mlp.down_proj", "model.layers.19.mlp.down_proj", "model.layers.21.mlp.down_proj", "model.layers.22.mlp.down_proj", . . . ] """ ``` To reproduce the vLLM error: ```bash vllm serve nm-testing/eldar-test ``` With this PR ------------ Models are loaded correctly without any errors.
1 parent cb3e73e commit 1867c25

File tree

1 file changed

+40
-1
lines changed
  • vllm/model_executor/layers/quantization/compressed_tensors

1 file changed

+40
-1
lines changed

vllm/model_executor/layers/quantization/compressed_tensors/utils.py

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ def find_matched_target(layer_name: Optional[str], module: Module,
103103

104104
matched_target = (_find_first_match(layer_name, targets)
105105
or _find_first_match(module.__class__.__name__, targets,
106-
True))
106+
True)
107+
or _match_fused_layer(layer_name, targets))
107108

108109
if matched_target is None:
109110
raise ValueError(f"Unable to find matching target for {module} in the "
@@ -152,3 +153,41 @@ def _is_equal_or_regex_match(value: str,
152153
elif target == value:
153154
return True
154155
return False
156+
157+
158+
def _match_fused_layer(layer_name: str,
159+
target_layers: Iterable[str]) -> Optional[str]:
160+
"""
161+
Match a fused layer name to its corresponding individual layer in
162+
target_layers.
163+
164+
Examples:
165+
layer_name = "model.layers.0.self_attn.qkv_proj"
166+
target_layers = ["model.layers.0.self_attn.q_proj",
167+
"model.layers.0.self_attn.k_proj",
168+
"model.layers.0.self_attn.v_proj"]
169+
"""
170+
# Split into parent path and layer type
171+
# e.g., "model.layers.0.self_attn" and "qkv_proj"
172+
parent_path = ".".join(layer_name.split(".")[:-1])
173+
layer_type = layer_name.split(".")[-1]
174+
175+
if layer_type not in FUSED_LAYER_NAME_MAPPING:
176+
return None
177+
178+
possible_layer_types = FUSED_LAYER_NAME_MAPPING[layer_type]
179+
180+
# Look for a target layer that:
181+
# 1. Has the same parent path
182+
# 2. Ends with one of the possible individual layer types
183+
for target in target_layers:
184+
is_same_parent = parent_path in target
185+
is_matching_type = any(type_suffix in target
186+
for type_suffix in possible_layer_types)
187+
188+
if is_same_parent and is_matching_type and all(
189+
'.'.join([parent_path, type_suffix])
190+
for type_suffix in possible_layer_types):
191+
return target
192+
193+
return None

0 commit comments

Comments
 (0)