metal : use FA-vec kernel up to batch size 20

ggerganov · ggerganov · commit 78d70223c3b3 · 2025-05-13T10:38:06.000+03:00
ggml-ci
diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m
@@ -4358,7 +4358,7 @@ static bool ggml_metal_encode_node(
                 // TODO: add vec kernels for (ne00%64 == 0) and maybe also for (ne00%32 == 0)
                 //       for now avoiding mainly to keep the number of templates/kernels a bit lower
                 //       these are now trivial to add after: https://github.com/ggml-org/llama.cpp/pull/12612
-                if (ne01 >= 4 || (ne00%128 != 0 && ne00 != 96 && ne00 != 192 && ne00 != 576)) {
+                if (ne01 >= 20 || (ne00%128 != 0 && ne00 != 96 && ne00 != 192 && ne00 != 576)) {
                     switch (src1->type) {
                         case GGML_TYPE_F16:
                             {

Original file line number	Diff line number	Diff line change
`@@ -4358,7 +4358,7 @@ static bool ggml_metal_encode_node(`
`4358`	`4358`	`// TODO: add vec kernels for (ne00%64 == 0) and maybe also for (ne00%32 == 0)`
`4359`	`4359`	`// for now avoiding mainly to keep the number of templates/kernels a bit lower`
`4360`	`4360`	`// these are now trivial to add after: https://github.com/ggml-org/llama.cpp/pull/12612`
`4361`		`- if (ne01 >= 4 \|\| (ne00%128 != 0 && ne00 != 96 && ne00 != 192 && ne00 != 576)) {`
	`4361`	`+ if (ne01 >= 20 \|\| (ne00%128 != 0 && ne00 != 96 && ne00 != 192 && ne00 != 576)) {`
`4362`	`4362`	`switch (src1->type) {`
`4363`	`4363`	`case GGML_TYPE_F16:`
`4364`	`4364`	`{`