Skip to content

Commit df3c066

Browse files
committed
opencl: do not take the vectorized f16 mrow GEMV path on an unaligned row stride
The multi-row f16 decode GEMV casts the src0 row pointer to half4 (8-byte) and, in the register-blocked h8 variants, to half8 (16-byte). The dispatch gate only requires ne00 % 4 == 0, which constrains the number of elements in a row -- not the byte stride between rows. nb01/nb02/nb03 and the view offset of a permuted or strided src0 are unconstrained, so a row can land 2-byte aligned and the vector load is then undefined behaviour. Same class as the f16/bf16 GEMV row-stride bug and the cvt.cl private-vector punning: Adreno is byte-addressable and tolerates the misaligned load, so it is invisible on the hardware this kernel currently runs on (the path is skipped on Intel today). Guard it rather than wait for the device that does not tolerate it. kernel_mul_mat_f16_f32_mrow now checks its own row pointer and falls back to the scalar loop, which has no alignment requirement. The register-blocked and half8 variants have no scalar path, so the host only selects them when the offset and every row stride are 8- (half4) or 16-byte (half8) aligned, and otherwise falls back to the base kernel. No functional or performance change on any shape dispatched today: every mrow shape in test-backend-ops is aligned (probed: a8=1/a16=1 on all 56 dispatches), so the guard is a no-op there. MUL_MAT 925/0 on the Adreno X2-90, unchanged.
1 parent a2547d8 commit df3c066

2 files changed

Lines changed: 23 additions & 5 deletions

File tree

ggml/src/ggml-opencl/ggml-opencl.cpp

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20530,17 +20530,28 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
2053020530
if (backend_ctx->f16_mrow && backend_ctx->gpu_family != INTEL &&
2053120531
backend_ctx->kernel_mul_mat_f16_f32_mrow != nullptr &&
2053220532
ne00 >= 128 && ne01 >= 8 && ne00 % 4 == 0 && ne00 <= 8192) {
20533+
// The register-blocked / half8 variants cast the src0 row pointer to
20534+
// half4 / half8 (8- and 16-byte loads) with no scalar fallback inside
20535+
// the kernel. ne00 % 4 == 0 constrains the element count per row, NOT
20536+
// the byte stride between rows: a permuted or strided src0 (or a view
20537+
// at an odd offset) can leave nb01/nb02/nb03 unaligned. Only take them
20538+
// when every row this dispatch touches is aligned; the base mrow kernel
20539+
// re-checks per row and falls back to its scalar loop.
20540+
const cl_ulong row_addr_bits = offset0 | nb01 | nb02 | nb03;
20541+
const bool aligned8 = (row_addr_bits & 7) == 0;
20542+
const bool aligned16 = (row_addr_bits & 15) == 0;
20543+
2053320544
// Register-blocked variants: each subgroup does RPT rows (more
2053420545
// weight loads in flight per lane). 8/16 use half8 (128-bit)
2053520546
// loads, gated on ne00 % 8 == 0.
2053620547
const int rpt = backend_ctx->f16_mrow_rpt;
20537-
if (rpt == 16 && ne00 % 8 == 0 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) {
20548+
if (rpt == 16 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2 != nullptr) {
2053820549
kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8r2;
20539-
} else if (rpt == 8 && ne00 % 8 == 0 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) {
20550+
} else if (rpt == 8 && ne00 % 8 == 0 && aligned16 && backend_ctx->kernel_mul_mat_f16_f32_mrow_h8 != nullptr) {
2054020551
kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_h8;
20541-
} else if (rpt == 4 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) {
20552+
} else if (rpt == 4 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r4 != nullptr) {
2054220553
kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r4;
20543-
} else if (rpt == 2 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) {
20554+
} else if (rpt == 2 && aligned8 && backend_ctx->kernel_mul_mat_f16_f32_mrow_r2 != nullptr) {
2054420555
kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow_r2;
2054520556
} else {
2054620557
kernel = backend_ctx->kernel_mul_mat_f16_f32_mrow;

ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,15 @@ kernel void kernel_mul_mat_f16_f32_mrow(
9292
ulong offset_src0 = r0*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
9393
global half * x = (global half *) (src0 + offset_src0);
9494

95+
// The vector path below casts the row pointer to half4, which must be 8-byte aligned.
96+
// A row address is r0*nb01 + ..., and a permuted or strided src0 leaves nb01/nb02/nb03
97+
// unconstrained -- ne00 % 4 == 0 bounds the element count per row, not the byte stride
98+
// between rows. Take the vector path only when this work-item's row is actually
99+
// aligned; the scalar loop below has no such requirement.
100+
const bool row_aligned = (((ulong) x) & 7) == 0;
101+
95102
float sumf = 0.0f;
96-
if (ne00 < 128) {
103+
if (ne00 < 128 || !row_aligned) {
97104
for (int i = lid; i < ne00; i += get_sub_group_size()) {
98105
sumf += (float) x[i] * ysh[i];
99106
}

0 commit comments

Comments
 (0)