Skip to content

Commit e9a5410

Browse files
lhezlhez
committed
opencl: initial q8_0 mv
1 parent 3edd87c commit e9a5410

File tree

3 files changed

+185
-1
lines changed

3 files changed

+185
-1
lines changed

‎ggml/src/ggml-opencl/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ set(GGML_OPENCL_KERNELS
8282
mul_mv_q4_0_f32_1d_8x_flat
8383
mul_mv_q4_0_f32_1d_16x_flat
8484
mul_mv_q6_k
85+
mul_mv_q8_0_f32
8586
mul_mv_mxfp4_f32
8687
mul_mv_mxfp4_f32_flat
8788
mul_mv_id_q4_0_f32_8x_flat

‎ggml/src/ggml-opencl/ggml-opencl.cpp‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,7 @@ struct ggml_backend_opencl_context {
367367
cl_program program_mul_mv_q4_0_f32_1d_8x_flat;
368368
cl_program program_mul_mv_q4_0_f32_1d_16x_flat;
369369
cl_program program_mul_mv_q6_K;
370+
cl_program program_mul_mv_q8_0_f32;
370371
cl_program program_mul_mv_mxfp4_f32;
371372
cl_program program_mul_mv_mxfp4_f32_flat;
372373
cl_program program_mul_mv_f16_f16;
@@ -455,6 +456,7 @@ struct ggml_backend_opencl_context {
455456
cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat;
456457
cl_kernel kernel_mul_mv_q6_K_f32;
457458
cl_kernel kernel_mul_mv_mxfp4_f32, kernel_mul_mv_mxfp4_f32_flat;
459+
cl_kernel kernel_mul_mv_q8_0_f32;
458460
cl_kernel kernel_im2col_f32, kernel_im2col_f16;
459461
cl_kernel kernel_argsort_f32_i32;
460462
cl_kernel kernel_sum_rows_f32;
@@ -992,6 +994,22 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx, ggml_cl_ve
992994
GGML_LOG_CONT(".");
993995
}
994996

997+
// mul_mv_q8_0_f32
998+
{
999+
#ifdef GGML_OPENCL_EMBED_KERNELS
1000+
const std::string kernel_src {
1001+
#include "mul_mv_q8_0_f32.cl.h"
1002+
};
1003+
#else
1004+
const std::string kernel_src = read_file("mul_mv_q8_0_f32.cl");
1005+
#endif
1006+
backend_ctx->program_mul_mv_q8_0_f32 =
1007+
build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
1008+
1009+
CL_CHECK((backend_ctx->kernel_mul_mv_q8_0_f32 = clCreateKernel(backend_ctx->program_mul_mv_q8_0_f32, "kernel_mul_mv_q8_0_f32", &err), err));
1010+
GGML_LOG_CONT(".");
1011+
}
1012+
9951013
// mul_mv_mxfp4_f32
9961014
{
9971015
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -2807,6 +2825,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
28072825
} else if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_MXFP4 ||
28082826
op->src[0]->type == GGML_TYPE_Q6_K) {
28092827
return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
2828+
} else if (op->src[0]->type == GGML_TYPE_Q8_0) {
2829+
return op->src[1]->type == GGML_TYPE_F32;
28102830
}
28112831
return false;
28122832
case GGML_OP_MUL_MAT_ID:
@@ -6937,7 +6957,45 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
69376957
#endif // GGML_OPENCL_SOA_Q
69386958
break;
69396959
case GGML_TYPE_Q4_1:
6940-
case GGML_TYPE_Q8_0:
6960+
case GGML_TYPE_Q8_0: {
6961+
kernel = backend_ctx->kernel_mul_mv_q8_0_f32;
6962+
6963+
// nth0 - subgroup size
6964+
// nth1 - number of subgroups per workgroup
6965+
// ndst - number of output values per workgroup = output per subgroup * number of subgroups
6966+
if (backend_ctx->gpu_family == INTEL) {
6967+
nth0 = 16;
6968+
nth1 = 2;
6969+
ndst = nth1*4;
6970+
} else if (backend_ctx->gpu_family == ADRENO) {
6971+
nth0 = 64;
6972+
nth1 = 2;
6973+
ndst = nth1*4;
6974+
} else {
6975+
GGML_ASSERT(false && "TODO: Unknown GPU");
6976+
}
6977+
6978+
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
6979+
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
6980+
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
6981+
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
6982+
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
6983+
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
6984+
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
6985+
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
6986+
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
6987+
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
6988+
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
6989+
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
6990+
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
6991+
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
6992+
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
6993+
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
6994+
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
6995+
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
6996+
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
6997+
break;
6998+
}
69416999
case GGML_TYPE_Q2_K:
69427000
case GGML_TYPE_Q3_K:
69437001
case GGML_TYPE_Q4_K:

‎ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
2+
3+
#ifdef cl_intel_subgroups
4+
#pragma OPENCL EXTENSION cl_intel_subgroups : enable
5+
#else
6+
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
7+
#endif
8+
9+
#ifdef cl_intel_required_subgroup_size
10+
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11+
#define INTEL_GPU 1
12+
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13+
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14+
#elif defined(cl_qcom_reqd_sub_group_size)
15+
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16+
#define ADRENO_GPU 1
17+
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18+
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19+
#endif
20+
21+
#define QK8_0 32
22+
typedef struct {
23+
half d; // delta
24+
char qs[QK8_0]; // quants
25+
} block_q8_0;
26+
27+
#define NB_Q8_0 8
28+
29+
#ifdef INTEL_GPU
30+
#define N_R0_Q8_0 4 // number of rows each subgroup works on
31+
#define N_SG_Q8_0 2 // number of subgroups in a work group
32+
#define N_SIMDWIDTH 16 // subgroup size
33+
#elif defined (ADRENO_GPU)
34+
#define N_R0_Q8_0 4
35+
#define N_SG_Q8_0 2
36+
#define N_SIMDWIDTH 64
37+
#endif
38+
39+
#ifdef INTEL_GPU
40+
REQD_SUBGROUP_SIZE_16
41+
#elif defined (ADRENO_GPU)
42+
REQD_SUBGROUP_SIZE_64
43+
#endif
44+
kernel void kernel_mul_mv_q8_0_f32(
45+
global char * src0,
46+
ulong offset0,
47+
global char * src1,
48+
ulong offset1,
49+
global char * dst,
50+
ulong offsetd,
51+
int ne00,
52+
int ne01,
53+
ulong nb01,
54+
ulong nb02,
55+
ulong nb03,
56+
int ne12,
57+
ulong nb11,
58+
ulong nb12,
59+
ulong nb13,
60+
int ne0,
61+
int ne1,
62+
int r2,
63+
int r3
64+
) {
65+
src0 = (global char*)((global char*)src0 + offset0);
66+
src1 = (global char*)((global char*)src1 + offset1);
67+
dst = (global char*)((global char*)dst + offsetd);
68+
69+
int nb = ne00/QK8_0;
70+
71+
int r0 = get_group_id(0);
72+
int r1 = get_group_id(1);
73+
int im = get_group_id(2);
74+
75+
int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;
76+
77+
uint i12 = im%ne12;
78+
uint i13 = im/ne12;
79+
80+
ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;
81+
global float * y = (global float *) (src1 + offset_src1);
82+
83+
// pointers to src0 rows
84+
global block_q8_0 * ax[N_R0_Q8_0];
85+
for (int row = 0; row < N_R0_Q8_0; ++row) {
86+
ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
87+
ax[row] = (global block_q8_0 *) ((global char *) src0 + offset_src0);
88+
}
89+
90+
float yl[NB_Q8_0];
91+
float sumf[N_R0_Q8_0] = { 0.f };
92+
93+
const short ix = get_sub_group_local_id()/4;
94+
const short il = get_sub_group_local_id()%4;
95+
96+
global float * yb = y + ix*QK8_0 + il*NB_Q8_0;
97+
98+
// each thread handles NB_Q8_0 quants at a time
99+
for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {
100+
for (short i = 0; i < NB_Q8_0; ++i) {
101+
yl[i] = yb[i];
102+
}
103+
104+
for (short row = 0; row < N_R0_Q8_0; row++) {
105+
global char * qs = ax[row][ib].qs + il*NB_Q8_0;
106+
float sumq = 0.f;
107+
for (short iq = 0; iq < NB_Q8_0; ++iq) {
108+
sumq += qs[iq] * yl[iq];
109+
}
110+
sumf[row] += sumq*ax[row][ib].d;
111+
}
112+
113+
yb += N_SIMDWIDTH*NB_Q8_0;
114+
}
115+
116+
global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
117+
118+
for (int row = 0; row < N_R0_Q8_0; ++row) {
119+
float tot = sub_group_reduce_add(sumf[row]);
120+
121+
if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
122+
dst_f32[first_row + row] = tot;
123+
}
124+
}
125+
}

0 commit comments

Comments
 (0)