Modify handling for Pixtral Large model params (#701)

* Modify handling for Pixtral Large model params.

* Fix multimodal_projector_bias to default to True if not in model config.json

Author: nintwentydo

exllamav2/architecture.py

@@ -312,7 +312,7 @@ class Params:
             })
             self.mmp.mlp_gate = False
             self.mmp.mlp_act_func = "gelu"
-            self.mmp.mlp_bias = True
+            self.mmp.mlp_bias = bool(read_config.get("multimodal_projector_bias", True))
 
         # Yi
 

exllamav2/config.py

@@ -476,9 +476,10 @@ def check_keys(archparams, prefix):
             self.vision_num_attention_heads = read(read_config, int, ["vision_config->num_attention_heads"], no_default)
             self.vision_num_key_value_heads = read(read_config, int, ["vision_config->num_key_value_heads"], self.vision_num_attention_heads)
             self.vision_num_key_value_groups = self.vision_num_attention_heads // self.vision_num_key_value_heads
+            self.multimodal_projector_bias = read(read_config, bool, ["multimodal_projector_bias"], True)
 
             self.vision_hidden_act = read(read_config, str, ["vision_config->hidden_act"], no_default)
-            self.vision_hidden_size = read(read_config, int, ["vision_config->image_size"], no_default)
+            self.vision_hidden_size = read(read_config, int, ["vision_config->hidden_size"], 1024)
             patch_size = read(read_config, int, ["vision_config->patch_size"], no_default)
             self.vision_rope_theta = read(read_config, int, ["vision_config->rope_theta"], no_default)
             self.vision_feature_layer = read(read_config, int, ["vision_feature_layer"], no_default)

exllamav2/vlm/mmprojector.py

@@ -27,6 +27,7 @@ def __init__(
                 in_features = cfg.vision_hidden_size,
                 out_features = cfg.hidden_size,
                 interm_features = cfg.hidden_size,
+                has_bias=cfg.multimodal_projector_bias,
                 has_norm = False,
                 has_residual = False,
             )