ATTN_NORM and FFN_NORM quantizable with specify

Author: Nexesenex

examples/quantize/quantize.cpp

@@ -102,7 +102,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
 //
 [[noreturn]]
 static void usage(const char * executable) {
-    printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--keep-split] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);
+    printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--attn-norm-type] [--ffn-norm-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--keep-split] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);
     printf("  --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n");
     printf("  --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n");
     printf("  --pure: Disable k-quant mixtures and quantize all tensors to the same type\n");
@@ -117,6 +117,8 @@ static void usage(const char * executable) {
     printf("      --attn-v-type ggml_type: use this ggml_type for the attn_v.weight tensor.\n");
     printf("      --attn-qkv-type ggml_type: use this ggml_type for the attn_qkv.weight tensor.\n");
     printf("      --attn-output-type ggml_type: use this ggml_type for the attn_output.weight tensor.\n");
+    printf("      --attn-norm-type ggml_type: use this ggml_type instead of F32 for the tiny attn_norm.weight tensor.\n");
+    printf("      --ffn-norm-type ggml_type: use this ggml_type instead of F32 for the tiny ffn_norm tensor.\n");
     printf("      --ffn-gate-type ggml_type: use this ggml_type for the ffn_gate tensor.\n");
     printf("      --ffn-down-type ggml_type: use this ggml_type for the ffn_down tensor.\n");
     printf("      --ffn-up-type ggml_type: use this ggml_type for the ffn_up tensor.\n\n");
@@ -314,6 +316,18 @@ int main(int argc, char ** argv) {
             } else {
                 usage(argv[0]);
             }
+        } else if (strcmp(argv[arg_idx], "--attn-norm-type") == 0) {
+            if (arg_idx < argc-1) {
+                params.attn_norm_type = parse_ggml_type(argv[++arg_idx]);
+            } else {
+                usage(argv[0]);
+            }
+        } else if (strcmp(argv[arg_idx], "--ffn-norm-type") == 0) {
+            if (arg_idx < argc-1) {
+                params.ffn_norm_type = parse_ggml_type(argv[++arg_idx]);
+            } else {
+                usage(argv[0]);
+            }
         } else if (strcmp(argv[arg_idx], "--ffn-gate-type") == 0) {
             if (arg_idx < argc-1) {
                 params.ffn_gate_type = parse_ggml_type(argv[++arg_idx]);

include/llama.h

@@ -358,6 +358,8 @@ extern "C" {
         enum ggml_type attn_v_type;          // attention value tensor type
         enum ggml_type attn_qkv_type;        // attention query-key-value tensor type
         enum ggml_type attn_output_type;     // attention output tensor type
+        enum ggml_type attn_norm_type;       // attention normalization tensor type
+        enum ggml_type ffn_norm_type;        // feedforward network normalization type
         enum ggml_type ffn_gate_type;        // feedforward network gate type
         enum ggml_type ffn_down_type;        // feedforward network down type
         enum ggml_type ffn_up_type;          // feedforward network up type

src/llama.cpp

@@ -16221,6 +16221,12 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
         else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XL && (use_more_bits(i_layer, n_layer))) new_type = GGML_TYPE_IQ4_XS;
         ++qs.i_ffn_up;
     }
+    else if (name.find("attn_norm.weight") != std::string::npos) {
+        if (ftype == LLAMA_FTYPE_CQS && qs.params->attn_norm_type < GGML_TYPE_COUNT) new_type = qs.params->attn_norm_type;
+    }
+    else if (name.find("ffn_norm") != std::string::npos) {
+        if (ftype == LLAMA_FTYPE_CQS && qs.params->ffn_norm_type < GGML_TYPE_COUNT) new_type = qs.params->ffn_norm_type;
+    }
 
     //    if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;
     //}
@@ -16649,6 +16655,12 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
             if (params->attn_output_type < GGML_TYPE_COUNT && strcmp(tensor->name, "attn_output.weight") == 0) {
                 new_type = params->attn_output_type;
             }
+            if (params->attn_norm_type < GGML_TYPE_COUNT && strcmp(tensor->name, "attn_norm.weight") == 0) {
+                new_type = params->attn_norm_type;
+            }
+            if (params->ffn_norm_type < GGML_TYPE_COUNT && strcmp(tensor->name, "ffn_norm") == 0) {
+                new_type = params->ffn_norm_type;
+            }
             if (params->ffn_gate_type < GGML_TYPE_COUNT && strcmp(tensor->name, "ffn_gate") == 0) {
                 new_type = params->ffn_gate_type;
             }
@@ -17065,6 +17077,8 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {
         /*.attn_v_type                 =*/ GGML_TYPE_COUNT,
         /*.attn_qkv_type               =*/ GGML_TYPE_COUNT,
         /*.attn_output_type            =*/ GGML_TYPE_COUNT,
+        /*.attn_norm_type              =*/ GGML_TYPE_COUNT,
+        /*.ffn_norm_type               =*/ GGML_TYPE_COUNT,
         /*.ffn_gate_type               =*/ GGML_TYPE_COUNT,
         /*.ffn_down_type               =*/ GGML_TYPE_COUNT,
         /*.ffn_up_type                 =*/ GGML_TYPE_COUNT,