Skip to content

Commit 3667a0a

Browse files
ravenouseravenouse
committed
Add example clip cli and enhance tensor name processing in Janus converter
1 parent b7fafb7 commit 3667a0a

File tree

2 files changed

+182
-35
lines changed

2 files changed

+182
-35
lines changed

‎examples/llava/clip-cli.cpp‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
//
2+
// Example usage of just the vision encoder (CLIP) part of the LLAVA codebase.
3+
// It loads a CLIP model (gguf file) and an image file,
4+
// computes the image embedding, and prints out (a few elements of) the embedding.
5+
//
6+
// Build and run (for example):
7+
// ./bin/llama-clip-cli -c model.gguf -i input.png --threads 1 --verbosity 1
8+
// ./bin/llama-clip-cli -c clip.gguf -i input.png --threads 1 --verbosity 1
9+
10+
#include "arg.h"
11+
#include "base64.hpp"
12+
#include "log.h"
13+
#include "common.h"
14+
#include "clip.h"
15+
#include "llava.h"
16+
#include "ggml.h"
17+
18+
#include <cstdio>
19+
#include <cstdlib>
20+
#include <cstring>
21+
#include <string>
22+
#include <vector>
23+
#include <algorithm>
24+
25+
// Structure to hold our command line parameters.
26+
struct vision_params {
27+
std::string clip_model; // Path to the CLIP model file (gguf)
28+
std::string image_file; // Path to the image file to process
29+
int n_threads = 1; // Number of CPU threads to use
30+
int verbosity = 1; // Verbosity level for model loading
31+
};
32+
33+
static void print_usage(const char* progname) {
34+
LOG("\nUsage: %s -c <clip_model_path> -i <image_file> [--threads <n_threads>] [--verbosity <level>]\n\n", progname);
35+
}
36+
37+
int main(int argc, char ** argv) {
38+
ggml_time_init();
39+
40+
vision_params params;
41+
42+
// Simple command line parsing
43+
if (argc < 5) {
44+
print_usage(argv[0]);
45+
return 1;
46+
}
47+
for (int i = 1; i < argc; i++) {
48+
std::string arg = argv[i];
49+
if (arg == "-c" || arg == "--clip") {
50+
if (i + 1 < argc) {
51+
params.clip_model = argv[++i];
52+
} else {
53+
print_usage(argv[0]);
54+
return 1;
55+
}
56+
} else if (arg == "-i" || arg == "--image") {
57+
if (i + 1 < argc) {
58+
params.image_file = argv[++i];
59+
} else {
60+
print_usage(argv[0]);
61+
return 1;
62+
}
63+
} else if (arg == "--threads") {
64+
if (i + 1 < argc) {
65+
params.n_threads = std::atoi(argv[++i]);
66+
} else {
67+
print_usage(argv[0]);
68+
return 1;
69+
}
70+
} else if (arg == "--verbosity") {
71+
if (i + 1 < argc) {
72+
params.verbosity = std::atoi(argv[++i]);
73+
} else {
74+
print_usage(argv[0]);
75+
return 1;
76+
}
77+
} else {
78+
// Unknown argument.
79+
print_usage(argv[0]);
80+
return 1;
81+
}
82+
}
83+
84+
if (params.clip_model.empty() || params.image_file.empty()) {
85+
print_usage(argv[0]);
86+
return 1;
87+
}
88+
89+
// Load the CLIP model.
90+
struct clip_ctx * ctx_clip = clip_model_load(params.clip_model.c_str(), params.verbosity);
91+
if (!ctx_clip) {
92+
LOG_ERR("Failed to load clip model from %s\n", params.clip_model.c_str());
93+
return 1;
94+
}
95+
LOG_INF("Clip model loaded from %s\n", params.clip_model.c_str());
96+
97+
// Load and process the image.
98+
llava_image_embed * embed = llava_image_embed_make_with_filename(ctx_clip, params.n_threads, params.image_file.c_str());
99+
if (!embed) {
100+
LOG_ERR("Failed to load or process image from %s\n", params.image_file.c_str());
101+
clip_free(ctx_clip);
102+
return 1;
103+
}
104+
LOG_INF("Image loaded and processed from %s\n", params.image_file.c_str());
105+
LOG_INF("Image embedding computed with %d positions.\n", embed->n_image_pos);
106+
int print_count = (embed->n_image_pos < 10 ? embed->n_image_pos : 10);
107+
LOG_INF("First %d elements: ", print_count);
108+
109+
for (int i = 0; i < print_count; i++) {
110+
LOG_INF("%f ", embed->embed[i]);
111+
}
112+
LOG_INF("\n");
113+
114+
llava_image_embed_free(embed);
115+
clip_free(ctx_clip);
116+
117+
return 0;
118+
}

‎examples/llava/convert_janus_encoder_to_gguf.py‎

Lines changed: 64 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,64 @@ def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: b
3737
return False
3838

3939

40-
def get_tensor_name(name: str) -> str:
41-
if "projection" in name:
42-
return name
43-
if "mm_projector" in name:
44-
name = name.replace("model.mm_projector", "mm")
45-
name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
46-
name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
47-
return name
40+
def get_tensor_name_from_janus(name: str) -> str:
41+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.attn\.qkv\.(weight|bias)$', r'v.blk.\1.attn_qkv.\2',name)
42+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.norm1\.(.*)$', r'v.blk.\1.ln1.\2', name)
43+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.attn\.proj\.(.*)$', r'v.blk.\1.attn_out.\2', name)
44+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.norm2\.(.*)$', r'v.blk.\1.ln2.\2', name)
45+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.mlp\.fc1\.(.*)$', r'v.blk.\1.ffn_down.\2', name)
46+
name = re.sub(r'^vision_tower\.blocks\.(\d+)\.mlp\.fc2\.(.*)$', r'v.blk.\1.ffn_up.\2', name)
47+
name = re.sub(r'^vision_tower\.patch_embed\.proj\.(.*)$', r'v.patch_embd.\1', name)
48+
name = re.sub(r'^vision_tower\.pos_embed$', r'v.position_embd.weight', name)
49+
name = re.sub(r'^vision_tower\.norm\.(weight|bias)$', r'v.post_ln.\1', name)
50+
51+
name = name.replace("vision_tower", "v")
52+
name = name.replace("text_model", "t")
53+
name = name.replace("vision_model", "v")
54+
name = name.replace("encoder.layers", "blk")
55+
name = name.replace("blocks", "blk")
56+
name = name.replace("embeddings.", "")
57+
name = name.replace("_proj", "")
58+
name = name.replace("self_attn.", "attn_")
59+
name = name.replace("layer_norm", "ln")
60+
name = name.replace("layernorm", "ln")
61+
name = name.replace("mlp.fc1", "ffn_down")
62+
name = name.replace("mlp.fc2", "ffn_up")
63+
name = name.replace("embedding", "embd")
64+
name = name.replace("final", "post")
65+
name = name.replace("layrnorm", "ln")
66+
67+
return name
68+
69+
70+
def process_and_save_tensor(tensor: torch.Tensor, new_name: str, ftype: int, fout) -> None:
71+
"""Process a tensor (squeeze, cast dtype, log) and save it to `fout`."""
72+
data = tensor.squeeze().numpy()
73+
n_dims = len(data.shape)
74+
ftype_str = {0: "f32", 1: "f16"}
4875

49-
return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
76+
ftype_cur = 0
77+
if n_dims == 4:
78+
print(f"tensor {new_name} is always saved in f16")
79+
data = data.astype(np.float16)
80+
ftype_cur = 1
81+
elif ftype == 1:
82+
if new_name.endswith(".weight") and n_dims == 2:
83+
print(" Converting to float16")
84+
data = data.astype(np.float16)
85+
ftype_cur = 1
86+
else:
87+
print(" Converting to float32")
88+
data = data.astype(np.float32)
89+
ftype_cur = 0
90+
else:
91+
if data.dtype != np.float32:
92+
print(" Converting to float32")
93+
data = data.astype(np.float32)
94+
ftype_cur = 0
5095

96+
print(f"{new_name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
97+
fout.add_tensor(new_name, data)
5198

5299
def bytes_to_unicode():
53100
"""
@@ -261,35 +308,17 @@ def bytes_to_unicode():
261308
print(f"skipping parameter: {name}")
262309
continue
263310

264-
name = get_tensor_name(name)
265-
data = data.squeeze().numpy()
311+
name = get_tensor_name_from_janus(name)
266312

267-
n_dims = len(data.shape)
313+
# Handle the qkv projection weights and biases
314+
if "qkv" in name:
315+
q_tensor, k_tensor, v_tensor = torch.chunk(data, 3, dim=0)
268316

269-
# ftype == 0 -> float32, ftype == 1 -> float16
270-
ftype_cur = 0
271-
if n_dims == 4:
272-
print(f"tensor {name} is always saved in f16")
273-
data = data.astype(np.float16)
274-
ftype_cur = 1
275-
elif ftype == 1:
276-
if name[-7:] == ".weight" and n_dims == 2:
277-
print(" Converting to float16")
278-
data = data.astype(np.float16)
279-
ftype_cur = 1
280-
else:
281-
print(" Converting to float32")
282-
data = data.astype(np.float32)
283-
ftype_cur = 0
317+
process_and_save_tensor(q_tensor, name.replace("qkv", "q"), ftype, fout)
318+
process_and_save_tensor(k_tensor, name.replace("qkv", "k"), ftype, fout)
319+
process_and_save_tensor(v_tensor, name.replace("qkv", "v"), ftype, fout)
284320
else:
285-
if data.dtype != np.float32:
286-
print(" Converting to float32")
287-
data = data.astype(np.float32)
288-
ftype_cur = 0
289-
290-
print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
291-
fout.add_tensor(name, data)
292-
321+
process_and_save_tensor(data, name, ftype, fout)
293322

294323
fout.write_header_to_file()
295324
fout.write_kv_data_to_file()

0 commit comments

Comments
 (0)