Skip to content

Commit 7d9d4e3

Browse files
committed
add debug_graph
1 parent 2ffafd5 commit 7d9d4e3

File tree

2 files changed

+89
-4
lines changed

2 files changed

+89
-4
lines changed

tools/mtmd/clip-impl.h

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
#include <climits>
66
#include <cstdarg>
7+
#include <cinttypes>
78
#include <string>
89
#include <map>
910
#include <sstream>
@@ -360,6 +361,70 @@ static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
360361
}
361362
}
362363

364+
//
365+
// debugging
366+
//
367+
368+
static void print_tensor_shape(ggml_tensor * t) {
369+
printf("%s.shape = [", t->name);
370+
for (int i = 0; i < ggml_n_dims(t); ++i) {
371+
printf("%" PRId64, t->ne[i]);
372+
if (i < ggml_n_dims(t) - 1) {
373+
printf(", ");
374+
}
375+
}
376+
printf("]\n");
377+
}
378+
379+
static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
380+
ggml_type type = t->type;
381+
int64_t * ne = t->ne;
382+
size_t * nb = t->nb;
383+
for (int64_t i3 = 0; i3 < ne[3]; i3++) {
384+
printf("%s.data: [\n", t->name);
385+
for (int64_t i2 = 0; i2 < ne[2]; i2++) {
386+
if (i2 == n && ne[2] > 2*n) {
387+
printf(" ..., \n");
388+
i2 = ne[2] - n;
389+
}
390+
printf(" [\n");
391+
for (int64_t i1 = 0; i1 < ne[1]; i1++) {
392+
if (i1 == n && ne[1] > 2*n) {
393+
printf(" ..., \n");
394+
i1 = ne[1] - n;
395+
}
396+
printf(" [");
397+
for (int64_t i0 = 0; i0 < ne[0]; i0++) {
398+
if (i0 == n && ne[0] > 2*n) {
399+
printf("..., ");
400+
i0 = ne[0] - n;
401+
}
402+
size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0];
403+
float v;
404+
if (type == GGML_TYPE_F16) {
405+
v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]);
406+
} else if (type == GGML_TYPE_F32) {
407+
v = *(float *) &data[i];
408+
} else if (type == GGML_TYPE_I32) {
409+
v = (float) *(int32_t *) &data[i];
410+
} else if (type == GGML_TYPE_I16) {
411+
v = (float) *(int16_t *) &data[i];
412+
} else if (type == GGML_TYPE_I8) {
413+
v = (float) *(int8_t *) &data[i];
414+
} else {
415+
GGML_ABORT("fatal error");
416+
}
417+
printf("%8.4f", v);
418+
if (i0 < ne[0] - 1) printf(", ");
419+
}
420+
printf("],\n");
421+
}
422+
printf(" ],\n");
423+
}
424+
printf(" ]\n");
425+
}
426+
}
427+
363428
//
364429
// API used internally with mtmd
365430
//

tools/mtmd/clip.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,12 @@ struct clip_ctx {
361361

362362
clip_image_size load_image_size;
363363

364+
// for debugging
365+
bool debug_graph = false;
366+
std::vector<ggml_tensor *> debug_print_tensors;
367+
364368
clip_ctx(clip_context_params & ctx_params) {
369+
debug_graph = std::getenv("MTMD_DEBUG_GRAPH") != nullptr;
365370
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
366371
if (!backend_cpu) {
367372
throw std::runtime_error("failed to initialize CPU backend");
@@ -1404,10 +1409,12 @@ struct clip_graph {
14041409
//
14051410

14061411
void cb(ggml_tensor * cur, const char * name, int il) const {
1407-
// TODO: implement this
1408-
GGML_UNUSED(cur);
1409-
GGML_UNUSED(name);
1410-
GGML_UNUSED(il);
1412+
if (ctx->debug_graph) {
1413+
std::string cur_name = il >= 0 ? std::string(name) + "_" + std::to_string(il) : name;
1414+
ggml_set_name(cur, cur_name.c_str());
1415+
ggml_set_output(cur);
1416+
ctx->debug_print_tensors.push_back(cur);
1417+
}
14111418
}
14121419

14131420
// build vision transformer (ViT) cgraph
@@ -3357,6 +3364,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
33573364
}
33583365

33593366
// build the inference graph
3367+
ctx->debug_print_tensors.clear();
33603368
ggml_backend_sched_reset(ctx->sched.get());
33613369
ggml_cgraph * gf = clip_image_build_graph(ctx, imgs);
33623370
ggml_backend_sched_alloc_graph(ctx->sched.get(), gf);
@@ -3675,6 +3683,18 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
36753683
return false;
36763684
}
36773685

3686+
// print debug nodes
3687+
if (ctx->debug_graph) {
3688+
LOG_INF("\n\n---\n\n");
3689+
LOG_INF("\n\nDebug graph:\n\n");
3690+
for (ggml_tensor * t : ctx->debug_print_tensors) {
3691+
std::vector<uint8_t> data(ggml_nbytes(t));
3692+
ggml_backend_tensor_get(t, data.data(), 0, ggml_nbytes(t));
3693+
print_tensor_shape(t);
3694+
print_tensor_data(t, data.data(), 3);
3695+
}
3696+
}
3697+
36783698
// the last node is the embedding tensor
36793699
ggml_tensor * embeddings = ggml_graph_node(gf, -1);
36803700

0 commit comments

Comments
 (0)