q4_4: general support for q4_4 and q8_4

Author: max-krasnyansky

convert_llama_ggml_to_gguf.py

@@ -43,6 +43,9 @@ class GGMLFType(IntEnum):
     MOSTLY_Q5_K_M        = 17
     MOSTLY_Q6_K          = 18
 
+    MOSTLY_Q4_4          = 32
+    MOSTLY_Q8_4          = 33
+
 
 class Hyperparameters:
     def __init__(self):

examples/quantize/quantize.cpp

@@ -49,6 +49,8 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
     { "Q4_0_4_4", LLAMA_FTYPE_MOSTLY_Q4_0_4_4, " 4.34G, +0.4685 ppl @ Llama-3-8B",  },
     { "Q4_0_4_8", LLAMA_FTYPE_MOSTLY_Q4_0_4_8, " 4.34G, +0.4685 ppl @ Llama-3-8B",  },
     { "Q4_0_8_8", LLAMA_FTYPE_MOSTLY_Q4_0_8_8, " 4.34G, +0.4685 ppl @ Llama-3-8B",  },
+    { "Q4_4",     LLAMA_FTYPE_MOSTLY_Q4_4,     " 3.35G, ??? ppl     @ LLaMA-v1-7B", },
+    { "Q8_4",     LLAMA_FTYPE_MOSTLY_Q8_4,     " 6.??G, ??? ppl     @ LLaMA-v1-7B", },
     { "F16",      LLAMA_FTYPE_MOSTLY_F16,      "14.00G, +0.0020 ppl @ Mistral-7B",  },
     { "BF16",     LLAMA_FTYPE_MOSTLY_BF16,     "14.00G, -0.0050 ppl @ Mistral-7B",  },
     { "F32",      LLAMA_FTYPE_ALL_F32,         "26.00G              @ 7B",          },

ggml/include/ggml.h

@@ -395,6 +395,8 @@ extern "C" {
         GGML_TYPE_Q4_0_4_4 = 31,
         GGML_TYPE_Q4_0_4_8 = 32,
         GGML_TYPE_Q4_0_8_8 = 33,
+        GGML_TYPE_Q4_4     = 34,
+        GGML_TYPE_Q8_4     = 35,
         GGML_TYPE_COUNT,
     };
 
@@ -439,6 +441,8 @@ extern "C" {
         GGML_FTYPE_MOSTLY_Q4_0_4_4 = 25, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q4_0_4_8 = 26, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q4_0_8_8 = 27, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q4_4     = 28, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q8_4     = 29, // except 1d tensors
     };
 
     // available tensor operations:

ggml/src/ggml-common.h

@@ -227,6 +227,20 @@ typedef struct {
 } block_q8_0x8;
 static_assert(sizeof(block_q8_0x8) == 8 * sizeof(ggml_half) + QK8_0 * 8, "wrong q8_0x8 block size/padding");
 
+#define QK4_4 128
+typedef struct {
+    ggml_half d;           // delta
+    uint8_t qs[QK4_4 / 2]; // nibbles / quants
+} block_q4_4;
+static_assert(sizeof(block_q4_4) == sizeof(ggml_half) + QK4_4 / 2, "wrong q4_4 block size/padding");
+
+#define QK8_4 128
+typedef struct {
+    ggml_half d;           // delta
+    int8_t  qs[QK8_4];     // quants
+} block_q8_4;
+static_assert(sizeof(block_q8_4) == sizeof(ggml_half) + QK8_4, "wrong q8_4 block size/padding");
+
 //
 // Super-block quantization structures
 //