Q4_0 Supported.

Author: QingtaoLi1

ggml/src/ggml-cpu/tmac/lut_mul_mat.cpp

@@ -413,7 +413,7 @@ static void ggml_tmac_tune_kernel_config(const struct ggml_tensor * tensor, int
                 }
 
                 for (int kfactor: kfactors) {
-                    if (kfactor < kernel_config.actk) {
+                    if ((kfactor < kernel_config.actk) || (kfactor * kernel_config.g > kernel_config.q_group_size)) {
                         continue;
                     }
 
@@ -455,7 +455,7 @@ static void ggml_tmac_tune_kernel_config(const struct ggml_tensor * tensor, int
 
         int largest_kfactor = 0;
         for (int kfactor: kfactors) {
-            if (kfactor < kernel_config.actk) {
+            if ((kfactor < kernel_config.actk) || (kfactor * kernel_config.g > kernel_config.q_group_size)) {
                 continue;
             }
             if (kfactor > largest_kfactor) {
@@ -468,8 +468,8 @@ static void ggml_tmac_tune_kernel_config(const struct ggml_tensor * tensor, int
 
     // Save the results
     insert_or_assign_tmac_kernel_config(M, K, bits, best_kcfg);
-    GGML_LOG_INFO("Tuned kernel config: M=%d, N=%d, K=%d, bm=%d, kfactor=%d, bits=%d, g=%d, ngroups_per_elem=%d, q_group_size=%d, act_group_size=%d\n",
-                    M, N, K, best_kcfg.bm, best_kcfg.kfactor, bits, best_kcfg.g, best_kcfg.ngroups_per_elem, best_kcfg.q_group_size, best_kcfg.act_group_size);
+    GGML_LOG_INFO("Tuned kernel config: M=%d, N=%d, K=%d, bm=%d, kfactor=%d, bits=%d, actk=%d, g=%d, ngroups_per_elem=%d, q_group_size=%d, act_group_size=%d\n",
+                    M, N, K, best_kcfg.bm, best_kcfg.kfactor, bits, best_kcfg.actk, best_kcfg.g, best_kcfg.ngroups_per_elem, best_kcfg.q_group_size, best_kcfg.act_group_size);
 }
 
 

ggml/src/ggml-cpu/tmac/tbl.cpp

@@ -834,14 +834,30 @@ void qgemm_lut_int8_g4(
     tbl_int32_reset(bm * sizeof(tmac_float_type) / sizeof(int32_t), (&(((int32_t*)CBits)[0])));
 
     int32_t k_outer_max = K / (kfactor * g);
+    int32_t scale_gs = q_group_size / (kfactor * g);
+    int32_t scale_idx_shfr = 0;
+    if (scale_gs == 1) {
+        scale_idx_shfr = 0;
+    } else if (scale_gs == 2) {
+        scale_idx_shfr = 1;
+    } else if (scale_gs == 4) {
+        scale_idx_shfr = 2;
+    } else if (scale_gs == 8) {
+        scale_idx_shfr = 3;
+    } else {
+        fprintf(stderr, "q_group_size=%d, kfactor=%d, g=%d\n", q_group_size, kfactor, g);
+        fprintf(stderr, "Unsupported scale group size over kfactor. Expected {1,2,4,8}, got %d.\n", scale_gs);
+        throw std::runtime_error("");
+    }
+
     for (int32_t k_outer = 0; k_outer < k_outer_max; k_outer++) {
         uint8_t * a = ((uint8_t *)A) + k_outer * bm * kfactor / ngroups_per_elem;
         tmac_float_type * scales = one_scale ? (tmac_float_type *)Scales :
-                              has_zero_point ? ((tmac_float_type *)Scales) + k_outer * m * 2:
-                                               ((tmac_float_type *)Scales) + k_outer * m;
+                              has_zero_point ? ((tmac_float_type *)Scales) + (k_outer >> scale_idx_shfr) * m * 2:
+                                               ((tmac_float_type *)Scales) + (k_outer >> scale_idx_shfr) * m;
         int8_t * lut = ((int8_t *)LUT) + k_outer * kfactor * int(pow(2, g));
-        tmac_float_type * lut_scales = ((tmac_float_type *)LUT_Scales) + (k_outer * q_group_size / act_group_size);  // k_outer * kfactor * g / act_group_size == k_outer
-        tmac_float_type * lut_biases = ((tmac_float_type *)LUT_Biases) + (k_outer * q_group_size / act_group_size);  // k_outer * kfactor * g / act_group_size == k_outer
+        tmac_float_type * lut_scales = ((tmac_float_type *)LUT_Scales) + (k_outer * kfactor * g / act_group_size);
+        tmac_float_type * lut_biases = ((tmac_float_type *)LUT_Biases) + (k_outer * kfactor * g / act_group_size);
 
         if (has_scale && kfactor == 8 && bits == 2 && actk == 8 && has_zero_point && !one_scale) {
             tbl_g4_int8_float_update_impl<true, 8, 2, 8, false, true, false>(