add clarify comment for norm eps

ngxson · web-flow · commit 1c0a25badc8c · 2025-05-10T15:54:34.000+02:00
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
@@ -907,6 +907,8 @@ struct clip_graph {
 
         // projector (always using GELU activation)
         {
+            // projector LayerNorm uses pytorch's default eps = 1e-5
+            // ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L79
             cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
             cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);
             cur = ggml_add(ctx0, cur, model.mm_1_b);

Original file line number	Diff line number	Diff line change
`@@ -907,6 +907,8 @@ struct clip_graph {`
`907`	`907`
`908`	`908`	`// projector (always using GELU activation)`
`909`	`909`	`{`
	`910`	`+ // projector LayerNorm uses pytorch's default eps = 1e-5`
	`911`	`+ // ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L79`
`910`	`912`	`cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);`
`911`	`913`	`cur = ggml_mul_mat(ctx0, model.mm_1_w, cur);`
`912`	`914`	`cur = ggml_add(ctx0, cur, model.mm_1_b);`