[examples] fix vision_tower/multi_modal_projector regexes (#1871)

SUMMARY:
Resolves #1652

Our multimodal examples all ignore `"re:vision_tower.*"`, but this
misses cases where the name is prefixed with something else (e.g.
`model.vision_tower`). This PR loosens the regexes to allow for anything
to precede `vision_tower` or `multi_modal_projector` and still be caught
by the ignore. Layers beginning with `vision_tower`, without a prefix,
will still be caught.

Also some formatting fixes, which must not be included on `examples/` as
part of ci/cd checks.

TEST PLAN:
Running `llm-compressor/examples/multimodal_vision/mistral3_example.py`
on latest main shows we are quantizing layers we don't want to be:
```
2025-09-26T20:02:43.571160+0000 | compress_modules | INFO - Quantizing model.vision_tower.transformer.layers.4.feed_forward.gate_proj using 512 samples
```

After these changes, those don't appear in the logs

---------

Signed-off-by: Brian Dellabetta <[email protected]>
Co-authored-by: Fynn Schmitt-Ulms <[email protected]>

Author: brian-dellabetta

examples/multimodal_vision/README.md

@@ -37,7 +37,7 @@ recipe = [
         targets="Linear",
         scheme="W4A16",
         sequential_targets=["MistralDecoderLayer"],
-        ignore=["re:.*lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"],
+        ignore=["re:.*lm_head", "re:.*vision_tower.*", "re:.*multi_modal_projector.*"],
     ),
 ]
 ```

examples/multimodal_vision/llama4_example.py

@@ -52,9 +52,11 @@ def preprocess_function(example):
 def data_collator(batch):
     assert len(batch) == 1
     return {
-        key: torch.tensor(value)
-        if key != "pixel_values"
-        else torch.tensor(value, dtype=torch.bfloat16).squeeze(0)
+        key: (
+            torch.tensor(value)
+            if key != "pixel_values"
+            else torch.tensor(value, dtype=torch.bfloat16).squeeze(0)
+        )
         for key, value in batch[0].items()
     }
 
@@ -67,8 +69,8 @@ def data_collator(batch):
         "re:.*lm_head",
         "re:.*self_attn",
         "re:.*router",
-        "re:vision_model.*",
-        "re:multi_modal_projector.*",
+        "re:.*vision_model.*",
+        "re:.*multi_modal_projector.*",
         "Llama4TextAttention",
     ],
 )

examples/multimodal_vision/llava_example.py

@@ -30,7 +30,7 @@ def data_collator(batch):
     GPTQModifier(
         targets="Linear",
         scheme="W4A16",
-        ignore=["re:.*lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"],
+        ignore=["re:.*lm_head", "re:.*vision_tower.*", "re:.*multi_modal_projector.*"],
     ),
 ]
 

examples/multimodal_vision/mistral3_example.py

@@ -31,9 +31,11 @@
 def data_collator(batch):
     assert len(batch) == 1
     return {
-        key: torch.tensor(value)
-        if key != "pixel_values"
-        else torch.tensor(value, dtype=model.dtype)
+        key: (
+            torch.tensor(value)
+            if key != "pixel_values"
+            else torch.tensor(value, dtype=model.dtype)
+        )
         for key, value in batch[0].items()
     }
 
@@ -43,7 +45,7 @@ def data_collator(batch):
     GPTQModifier(
         targets="Linear",
         scheme="W4A16",
-        ignore=["re:.*lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"],
+        ignore=["re:.*lm_head", "re:.*vision_tower.*", "re:.*multi_modal_projector.*"],
     ),
 ]
 

examples/multimodal_vision/mllama_example.py

@@ -30,7 +30,7 @@ def data_collator(batch):
     GPTQModifier(
         targets="Linear",
         scheme="W4A16",
-        ignore=["re:.*lm_head", "re:multi_modal_projector.*", "re:vision_model.*"],
+        ignore=["re:.*lm_head", "re:.*multi_modal_projector.*", "re:.*vision_model.*"],
     ),
 ]
 

examples/multimodal_vision/pixtral_example.py

@@ -36,7 +36,7 @@ def data_collator(batch):
     GPTQModifier(
         targets="Linear",
         scheme="W4A16",
-        ignore=["re:.*lm_head", "re:vision_tower.*", "re:multi_modal_projector.*"],
+        ignore=["re:.*lm_head", "re:.*vision_tower.*", "re:.*multi_modal_projector.*"],
     ),
 ]
 

examples/quantization_w4a4_fp4/llama4_example.py

@@ -52,9 +52,11 @@ def preprocess_function(example):
 def data_collator(batch):
     assert len(batch) == 1
     return {
-        key: torch.tensor(value)
-        if key != "pixel_values"
-        else torch.tensor(value, dtype=torch.bfloat16).squeeze(0)
+        key: (
+            torch.tensor(value)
+            if key != "pixel_values"
+            else torch.tensor(value, dtype=torch.bfloat16).squeeze(0)
+        )
         for key, value in batch[0].items()
     }
 
@@ -67,8 +69,8 @@ def data_collator(batch):
         "re:.*lm_head",
         "re:.*self_attn",
         "re:.*router",
-        "re:vision_model.*",
-        "re:multi_modal_projector.*",
+        "re:.*vision_model.*",
+        "re:.*multi_modal_projector.*",
         "Llama4TextAttention",
     ],
 )

examples/quantization_w8a8_fp8/llama3.2_vision_example.py

@@ -17,7 +17,7 @@
 recipe = QuantizationModifier(
     targets="Linear",
     scheme="FP8_DYNAMIC",
-    ignore=["re:.*lm_head", "re:multi_modal_projector.*", "re:vision_model.*"],
+    ignore=["re:.*lm_head", "re:.*multi_modal_projector.*", "re:.*vision_model.*"],
 )
 
 # Apply quantization and save to disk in compressed-tensors format.

examples/quantization_w8a8_fp8/llama4_fp8_block_example.py

@@ -22,8 +22,8 @@
         "re:.*lm_head",
         "re:.*self_attn",
         "re:.*router",
-        "re:vision_model.*",
-        "re:multi_modal_projector.*",
+        "re:.*vision_model.*",
+        "re:.*multi_modal_projector.*",
         "Llama4TextAttention",
     ],
 )

examples/quantization_w8a8_fp8/llava1.5_example.py

@@ -17,7 +17,7 @@
 recipe = QuantizationModifier(
     targets="Linear",
     scheme="FP8_DYNAMIC",
-    ignore=["re:.*lm_head", "re:multi_modal_projector.*", "re:vision_tower.*"],
+    ignore=["re:.*lm_head", "re:.*multi_modal_projector.*", "re:.*vision_tower.*"],
 )
 
 # Apply quantization and save to disk in compressed-tensors format.