Update llama-quant.cpp

danielhanchen · danielhanchen · commit 6fb3dfef5132 · 2025-01-26T03:58:31.000-08:00
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
@@ -212,6 +212,7 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, ggml_type new_t
             // Layers 0, 1, 2 are Dense so Q4_K
             // 3, 4, 5 left as Q2_K
             if (is_one_bit) {
+                // 3, 4, 5, 6, 7, 8 left as 2.06 bpw
                 if (i_layer < 9) new_type = GGML_TYPE_IQ2_XXS; // 2.06 bpw
             }
             else {

Original file line number	Diff line number	Diff line change
`@@ -212,6 +212,7 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, ggml_type new_t`
`212`	`212`	`// Layers 0, 1, 2 are Dense so Q4_K`
`213`	`213`	`// 3, 4, 5 left as Q2_K`
`214`	`214`	`if (is_one_bit) {`
	`215`	`+ // 3, 4, 5, 6, 7, 8 left as 2.06 bpw`
`215`	`216`	`if (i_layer < 9) new_type = GGML_TYPE_IQ2_XXS; // 2.06 bpw`
`216`	`217`	`}`
`217`	`218`	`else {`