Skip to content

Commit 8b3d319

Browse files
committed
clip-vit: corrected cls_embd concat
1 parent cec9a5c commit 8b3d319

File tree

1 file changed

+4
-8
lines changed

1 file changed

+4
-8
lines changed

tools/mtmd/clip.cpp

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1538,15 +1538,11 @@ struct clip_graph {
15381538
GGML_ASSERT(model.position_embeddings != nullptr);
15391539

15401540
const int n_pos = n_patches + 1;
1541-
ggml_tensor * inp =
1542-
ggml_cont_3d(ctx0, ggml_dup_tensor(ctx0, patch_embeds), patch_embeds->ne[0], n_patches_x, n_patches_y);
1543-
1544-
auto inp_n_elems = ggml_nelements(inp);
1545-
GGML_ASSERT(inp_n_elems == inp->ne[0] * inp->ne[1] * inp->ne[2]);
1546-
inp = ggml_permute(ctx0, inp, 2, 1,0,3); // [n_patches, n_embd]
1541+
ggml_tensor * inp = ggml_permute(ctx0, patch_embeds,2,1,0,3);
15471542
inp = ggml_cont(ctx0, inp);
1548-
GGML_ASSERT(ggml_nelements(inp) == n_patches_x*patch_size*4*768);
1549-
inp= ggml_reshape_2d(ctx0,inp,n_patches_x*patch_size, 4*768);
1543+
inp = ggml_reshape_2d(ctx0, inp, n_embd, n_patches);
1544+
1545+
15501546

15511547
// add CLS token
15521548
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);

0 commit comments

Comments
 (0)