img pre processing

Author: ngxson

convert_hf_to_gguf.py

@@ -1584,8 +1584,13 @@ def set_gguf_parameters(self):
             self.gguf_writer.add_vision_clip_feed_forward_length(self.vparams["intermediate_size"])
             self.gguf_writer.add_vision_clip_head_count(self.vparams["num_attention_heads"])
             # TODO: should not hardcode these, but they are currently missing from config.json
+            self.gguf_writer.add_vision_clip_projector_type(gguf.constants.CLIPProjectorType.MLP)
             self.gguf_writer.add_vision_clip_max_position_embeddings(577)
             self.gguf_writer.add_vision_clip_layer_norm_epsilon(1e-05)
+            default_image_mean = [0.48145466, 0.4578275, 0.40821073]
+            default_image_std = [0.26862954, 0.26130258, 0.27577711]
+            self.gguf_writer.add_vision_clip_image_mean(default_image_mean)
+            self.gguf_writer.add_vision_clip_image_std(default_image_std)
 
     @staticmethod
     def permute(weights: Tensor, n_head: int, n_head_kv: int | None):

gguf-py/gguf/constants.py

@@ -196,6 +196,7 @@ class Clip:
             PROJECTION_DIM      = "vision.clip.projection_dim"
             USE_GELU            = "vision.clip.use_gelu"
             MAX_POS_EMBEDDING   = "vision.clip.max_position_embeddings"
+            PROJECTOR_TYPE      = "vision.clip.projector_type"
             HEAD_COUNT          = "vision.clip.attention.head_count"
             LAYERNORM_EPS       = "vision.clip.attention.layer_norm_epsilon"
 
@@ -1425,6 +1426,10 @@ class PoolingType(IntEnum):
     CLS  = 2
 
 
+class CLIPProjectorType(Enum):
+    MLP = 'mlp'
+
+
 class GGMLQuantizationType(IntEnum):
     F32     = 0
     F16     = 1

gguf-py/gguf/gguf_writer.py

@@ -26,6 +26,7 @@
     RopeScalingType,
     PoolingType,
     TokenType,
+    CLIPProjectorType,
 )
 
 from .quants import quant_shape_from_byte_shape
@@ -844,9 +845,18 @@ def add_vision_clip_head_count(self, value: int) -> None:
     def add_vision_clip_max_position_embeddings(self, value: int) -> None:
         self.add_uint32(Keys.Vision.Clip.MAX_POS_EMBEDDING, value)
 
+    def add_vision_clip_projector_type(self, value: CLIPProjectorType) -> None:
+        self.add_string(Keys.Vision.Clip.PROJECTOR_TYPE, value.value)
+
     def add_vision_clip_layer_norm_epsilon(self, value: float) -> None:
         self.add_float32(Keys.Vision.Clip.LAYERNORM_EPS, value)
 
+    def add_vision_clip_image_mean(self, value: Sequence[float]) -> None:
+        self.add_array(Keys.Vision.IMAGE_MEAN, value)
+
+    def add_vision_clip_image_std(self, value: Sequence[float]) -> None:
+        self.add_array(Keys.Vision.IMAGE_STD, value)
+
     def add_chat_template(self, value: str | Sequence[Mapping[str, str]]) -> None:
         if not isinstance(value, str):
             template_default = None