Skip to content

Commit 6617024

Browse files
author
liyang
committed
Refactor JinaCLIP vision mmproj mapping to use tensor_mapping table
1 parent 49c98be commit 6617024

File tree

3 files changed

+51
-96
lines changed

3 files changed

+51
-96
lines changed

‎convert_hf_to_gguf.py‎

Lines changed: 11 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -1531,7 +1531,9 @@ class MmprojModel(ModelBase):
15311531
preprocessor_config: dict[str, Any]
15321532
global_config: dict[str, Any]
15331533

1534-
n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]
1534+
# Prefer explicit "layers" (e.g. JinaCLIP),
1535+
# keep legacy keys for other models.
1536+
n_block_keys = ["layers", "n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]
15351537

15361538
has_vision_encoder: bool = True # by default
15371539
has_audio_encoder: bool = False
@@ -6775,6 +6777,11 @@ def __init__(self, *args, **kwargs):
67756777
with open(config_path, encoding="utf-8") as f:
67766778
self.vision_config = json.load(f)
67776779

6780+
def get_vision_config(self) -> dict[str, Any] | None:
6781+
# For JinaCLIPVisionModel, the top-level AutoConfig dict is already
6782+
# the vision-only configuration.
6783+
return self.global_config
6784+
67786785
def set_vocab(self):
67796786
# Vision encoder doesn't need vocabulary
67806787
pass
@@ -6832,73 +6839,10 @@ def set_gguf_parameters(self):
68326839
def _strip_vm_prefix(self, name: str) -> str:
68336840
return name[len('vision_model.'):] if name.startswith('vision_model.') else name
68346841

6835-
def _map_block_tensor(self, layer: int, rest: str, data_torch: Tensor, name: str) -> list[tuple[str, Tensor]] | None:
6836-
parts = rest.split('.')
6837-
# layer norms
6838-
if rest.startswith('norm1.'):
6839-
suffix = parts[-1]
6840-
return [(f'v.blk.{layer}.ln1.{suffix}', data_torch)]
6841-
if rest.startswith('norm2.'):
6842-
suffix = parts[-1]
6843-
return [(f'v.blk.{layer}.ln2.{suffix}', data_torch)]
6844-
if rest.startswith('attn.inner_attn_ln.'):
6845-
suffix = parts[-1]
6846-
return [(f'v.blk.{layer}.attn_ln.{suffix}', data_torch)]
6847-
6848-
if rest == 'attn.q_bias':
6849-
return [(f'v.blk.{layer}.attn_q.bias', data_torch)]
6850-
if rest == 'attn.v_bias':
6851-
return [(f'v.blk.{layer}.attn_v.bias', data_torch)]
6852-
6853-
if rest.startswith('attn.q_proj.'):
6854-
suffix = parts[-1]
6855-
return [(f'v.blk.{layer}.attn_q.{suffix}', data_torch)]
6856-
if rest.startswith('attn.k_proj.'):
6857-
suffix = parts[-1]
6858-
return [(f'v.blk.{layer}.attn_k.{suffix}', data_torch)]
6859-
if rest.startswith('attn.v_proj.'):
6860-
suffix = parts[-1]
6861-
return [(f'v.blk.{layer}.attn_v.{suffix}', data_torch)]
6862-
if rest.startswith('attn.proj.'):
6863-
suffix = parts[-1]
6864-
return [(f'v.blk.{layer}.attn_out.{suffix}', data_torch)]
6865-
6866-
# MLP
6867-
if rest.startswith('mlp.w1.'):
6868-
suffix = parts[-1]
6869-
return [(f'v.blk.{layer}.ffn_gate.{suffix}', data_torch)]
6870-
if rest.startswith('mlp.w2.'):
6871-
suffix = parts[-1]
6872-
return [(f'v.blk.{layer}.ffn_up.{suffix}', data_torch)]
6873-
if rest.startswith('mlp.w3.'):
6874-
suffix = parts[-1]
6875-
return [(f'v.blk.{layer}.ffn_down.{suffix}', data_torch)]
6876-
if rest.startswith('mlp.ffn_ln.'):
6877-
suffix = parts[-1]
6878-
return [(f'v.blk.{layer}.ffn_norm.{suffix}', data_torch)]
6879-
if rest.startswith('mlp.fc1.'):
6880-
suffix = parts[-1]
6881-
return [(f'v.blk.{layer}.ffn_up.{suffix}', data_torch)]
6882-
if rest.startswith('mlp.fc2.'):
6883-
suffix = parts[-1]
6884-
return [(f'v.blk.{layer}.ffn_down.{suffix}', data_torch)]
6885-
return None
6886-
68876842
def map_tensor_name(self, name: str, try_suffixes: Sequence[str] = (".weight", ".bias")) -> str:
6888-
"""Prefer base table-driven mapping; keep Jina-specific targets if already mapped; fallback to legacy mapper."""
6889-
# Already a GGUF target name (e.g., "v.*" or "mm.*"): return as-is
68906843
if name.startswith('v.') or name.startswith('mm.'):
68916844
return name
6892-
# Try the base mapping first
6893-
try:
6894-
return super().map_tensor_name(name, try_suffixes=try_suffixes)
6895-
except Exception:
6896-
# Fallback to legacy Jina-specific mapper for any remaining edge keys
6897-
if hasattr(self, "_map_jinaclip_tensor_name"):
6898-
mapped = self._map_jinaclip_tensor_name(name) # type: ignore[attr-defined]
6899-
if mapped:
6900-
return mapped
6901-
return name
6845+
return super().map_tensor_name(name, try_suffixes=try_suffixes)
69026846

69036847
def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
69046848
yielded_any = False
@@ -6937,39 +6881,10 @@ def _should_be_f32(self, gguf_name: str) -> bool:
69376881
return any(p in gguf_name for p in patterns)
69386882

69396883
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
6940-
del bid # unused
6941-
6942-
src = name
6943-
if src.startswith('v.') or src.startswith('mm.'):
6944-
return [(src, data_torch)]
6945-
6946-
# Drop 'vision_model.' prefix if present
6947-
src_no_vm = self._strip_vm_prefix(src)
6948-
6949-
# Top-level direct mappings — use gguf constants directly for canonical names
6950-
if src_no_vm == 'cls_token':
6951-
base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_CLS]
6952-
return [(base, data_torch)]
6953-
if src_no_vm.startswith('patch_embed.proj.'):
6954-
suffix = src_no_vm.split('.')[-1]
6955-
base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH]
6956-
return [(f'{base}.{suffix}', data_torch)]
6957-
if src_no_vm == 'pos_embed':
6884+
# keep only pos_embed special case (no .weight suffix); all other tensors use table-driven mapping
6885+
if name == 'pos_embed':
69586886
pos_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_POS] + '.weight'
69596887
return [(pos_name, data_torch)]
6960-
if src_no_vm.startswith('norm.'):
6961-
suffix = src_no_vm.split('.')[-1]
6962-
base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_POST_NORM]
6963-
return [(f'{base}.{suffix}', data_torch)]
6964-
6965-
if src_no_vm.startswith('blocks.'):
6966-
parts = src_no_vm.split('.')
6967-
if len(parts) >= 3 and parts[1].isdigit():
6968-
layer = int(parts[1])
6969-
rest = '.'.join(parts[2:])
6970-
mapped = self._map_block_tensor(layer, rest, data_torch, name)
6971-
if mapped is not None:
6972-
return mapped
69736888

69746889
try:
69756890
return [(self.map_tensor_name(name), data_torch)]

‎gguf-py/gguf/constants.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,9 +634,13 @@ class MODEL_TENSOR(IntEnum):
634634
V_ENC_ATTN_O = auto()
635635
V_ENC_ATTN_O_NORM = auto()
636636
V_ENC_POST_ATTN_NORM = auto()
637+
V_ENC_ATTN_LN = auto()
637638
V_ENC_FFN_UP = auto()
638639
V_ENC_FFN_GATE = auto()
639640
V_ENC_FFN_DOWN = auto()
641+
V_ENC_FFN_NORM = auto()
642+
V_ENC_ATTN_Q_BIAS = auto()
643+
V_ENC_ATTN_V_BIAS = auto()
640644
V_LAYER_SCALE_1 = auto()
641645
V_LAYER_SCALE_2 = auto()
642646
V_PRE_NORM = auto()
@@ -1002,9 +1006,13 @@ class MODEL_TENSOR(IntEnum):
10021006
MODEL_TENSOR.V_ENC_ATTN_O: "v.blk.{bid}.attn_out",
10031007
MODEL_TENSOR.V_ENC_ATTN_O_NORM: "v.blk.{bid}.attn_out_norm",
10041008
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: "v.blk.{bid}.ln2",
1009+
MODEL_TENSOR.V_ENC_ATTN_LN: "v.blk.{bid}.attn_ln",
10051010
MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up",
10061011
MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate",
10071012
MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down",
1013+
MODEL_TENSOR.V_ENC_FFN_NORM: "v.blk.{bid}.ffn_norm",
1014+
MODEL_TENSOR.V_ENC_ATTN_Q_BIAS: "v.blk.{bid}.attn_q.bias",
1015+
MODEL_TENSOR.V_ENC_ATTN_V_BIAS: "v.blk.{bid}.attn_v.bias",
10081016
MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1",
10091017
MODEL_TENSOR.V_LAYER_SCALE_2: "v.blk.{bid}.ls2",
10101018
MODEL_TENSOR.V_PRE_NORM: "v.pre_ln",
@@ -1080,9 +1088,13 @@ class MODEL_TENSOR(IntEnum):
10801088
MODEL_TENSOR.V_ENC_ATTN_O,
10811089
MODEL_TENSOR.V_ENC_ATTN_O_NORM,
10821090
MODEL_TENSOR.V_ENC_POST_ATTN_NORM,
1091+
MODEL_TENSOR.V_ENC_ATTN_LN,
10831092
MODEL_TENSOR.V_ENC_FFN_UP,
10841093
MODEL_TENSOR.V_ENC_FFN_GATE,
10851094
MODEL_TENSOR.V_ENC_FFN_DOWN,
1095+
MODEL_TENSOR.V_ENC_FFN_NORM,
1096+
MODEL_TENSOR.V_ENC_ATTN_Q_BIAS,
1097+
MODEL_TENSOR.V_ENC_ATTN_V_BIAS,
10861098
MODEL_TENSOR.V_LAYER_SCALE_1,
10871099
MODEL_TENSOR.V_LAYER_SCALE_2,
10881100
MODEL_TENSOR.V_PRE_NORM,

‎gguf-py/gguf/tensor_mapping.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,7 @@ class TensorNameMap:
12021202
"model.vision_tower.embeddings.cls_token", # Intern-S1
12031203
"vision_model.class_embedding", # llama 4
12041204
"model.vision.patch_embedding.cls_embedding", # cogvlm
1205+
"cls_token", # JinaCLIP v2 vision
12051206
),
12061207

12071208
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
@@ -1215,6 +1216,7 @@ class TensorNameMap:
12151216
"visual.patch_embed.proj", # qwen2vl
12161217
"vision_tower.patch_embed.proj", # kimi-vl
12171218
"model.vision.patch_embedding.proj", # cogvlm
1219+
"patch_embed.proj", # JinaCLIP v2 vision
12181220
),
12191221

12201222
MODEL_TENSOR.V_ENC_EMBD_POS: (
@@ -1243,6 +1245,7 @@ class TensorNameMap:
12431245
"vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
12441246
"visual.blocks.{bid}.attn.q", # qwen2vl, generated
12451247
"vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
1248+
"blocks.{bid}.attn.q_proj", # JinaCLIP v2 vision
12461249
),
12471250

12481251
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
@@ -1260,6 +1263,7 @@ class TensorNameMap:
12601263
"vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
12611264
"visual.blocks.{bid}.attn.k", # qwen2vl, generated
12621265
"vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
1266+
"blocks.{bid}.attn.k_proj", # JinaCLIP v2 vision
12631267
),
12641268

12651269
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
@@ -1277,6 +1281,7 @@ class TensorNameMap:
12771281
"vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
12781282
"visual.blocks.{bid}.attn.v", # qwen2vl, generated
12791283
"vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
1284+
"blocks.{bid}.attn.v_proj", # JinaCLIP v2 vision
12801285
),
12811286

12821287
MODEL_TENSOR.V_ENC_INPUT_NORM: (
@@ -1291,6 +1296,7 @@ class TensorNameMap:
12911296
"visual.blocks.{bid}.norm1", # qwen2vl
12921297
"vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
12931298
"model.vision.transformer.layers.{bid}.input_layernorm", # cogvlm
1299+
"blocks.{bid}.norm1", # JinaCLIP v2 vision
12941300
),
12951301

12961302
MODEL_TENSOR.V_ENC_ATTN_O: (
@@ -1306,6 +1312,7 @@ class TensorNameMap:
13061312
"visual.blocks.{bid}.attn.proj", # qwen2vl
13071313
"vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
13081314
"model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
1315+
"blocks.{bid}.attn.proj", # JinaCLIP v2 vision
13091316
),
13101317

13111318
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
@@ -1320,6 +1327,11 @@ class TensorNameMap:
13201327
"visual.blocks.{bid}.norm2", # qwen2vl
13211328
"vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
13221329
"model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
1330+
"blocks.{bid}.norm2", # JinaCLIP v2 vision
1331+
),
1332+
1333+
MODEL_TENSOR.V_ENC_ATTN_LN: (
1334+
"blocks.{bid}.attn.inner_attn_ln", # JinaCLIP v2 vision
13231335
),
13241336

13251337
MODEL_TENSOR.V_ENC_FFN_UP: (
@@ -1335,12 +1347,14 @@ class TensorNameMap:
13351347
"visual.blocks.{bid}.mlp.linear_fc1", # qwen3vl
13361348
"vision_tower.encoder.blocks.{bid}.mlp.fc0", # kimi-vl (fc0/fc1)
13371349
"model.vision.transformer.layers.{bid}.mlp.fc1", # cogvlm
1350+
"blocks.{bid}.mlp.w2", # JinaCLIP v2 vision (up)
13381351
),
13391352

13401353
MODEL_TENSOR.V_ENC_FFN_GATE: (
13411354
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
13421355
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
13431356
"visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
1357+
"blocks.{bid}.mlp.w1", # JinaCLIP v2 vision
13441358
),
13451359

13461360
MODEL_TENSOR.V_ENC_FFN_DOWN: (
@@ -1356,6 +1370,11 @@ class TensorNameMap:
13561370
"visual.blocks.{bid}.mlp.linear_fc2", # qwen3vl
13571371
"vision_tower.encoder.blocks.{bid}.mlp.fc1", # kimi-vl (fc0/fc1)
13581372
"model.vision.transformer.layers.{bid}.mlp.fc2", # cogvlm
1373+
"blocks.{bid}.mlp.w3", # JinaCLIP v2 vision (down)
1374+
),
1375+
1376+
MODEL_TENSOR.V_ENC_FFN_NORM: (
1377+
"blocks.{bid}.mlp.ffn_ln", # JinaCLIP v2 vision
13591378
),
13601379

13611380
MODEL_TENSOR.V_LAYER_SCALE_1: (
@@ -1368,6 +1387,14 @@ class TensorNameMap:
13681387
"model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1
13691388
),
13701389

1390+
MODEL_TENSOR.V_ENC_ATTN_Q_BIAS: (
1391+
"blocks.{bid}.attn.q_bias", # JinaCLIP v2 vision
1392+
),
1393+
1394+
MODEL_TENSOR.V_ENC_ATTN_V_BIAS: (
1395+
"blocks.{bid}.attn.v_bias", # JinaCLIP v2 vision
1396+
),
1397+
13711398
MODEL_TENSOR.V_PRE_NORM: (
13721399
"vision_tower.vision_model.pre_layrnorm",
13731400
"vision_tower.ln_pre", # pixtral-hf
@@ -1381,6 +1408,7 @@ class TensorNameMap:
13811408
"vision_model.layernorm_post", # llama4
13821409
"visual.merger.ln_q", # qwen2vl
13831410
"vision_tower.encoder.final_layernorm", # kimi-vl
1411+
"norm", # JinaCLIP v2 vision
13841412
),
13851413

13861414
MODEL_TENSOR.V_MM_INP_PROJ: (

0 commit comments

Comments
 (0)