From 3ff664d2a054a2649de589d360393ee4d26b3ea4 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Mon, 6 Jul 2026 15:28:55 -0700 Subject: [PATCH 1/3] opencl: fix garbled output for Q6_K weights with ne01 % 128 != 0 on Adreno Observed with granite-3.1-3b-a800m-instruct, whose vocab is an odd number. Route Q6_K dense mul_mat with ne01 % 128 != 0 off the noshuffle path: decode (ne1==1) uses the correct flat GEMV and the matching GEMM (ne1>1) falls back to CPU (the flat convert has no verified small-batch GEMM kernel for these shapes). All standard hidden/FFN/vocab dims are multiples of 128 and keep the noshuffle path. --- ggml/src/ggml-opencl/ggml-opencl.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 10e183530e74..3abe274f4999 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -6273,6 +6273,19 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) { // threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold. // note that this forces large M weights to use LM GEMM. + // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the + // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of + // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles. + // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe + // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every + // odd column of the transposed layout (gross garbage) and dropping the last row; + // other non-multiples over-dispatch and write past the end of dst. Route such + // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back + // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of + // 128 and keep the noshuffle path. + if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + return true; + } return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1; } @@ -6479,6 +6492,15 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // q6_K GEMM (ne1>1) for weights routed off the noshuffle path because + // ne01 % 128 != 0 (see use_flat_gemv_for_large_m_q6_K): the flat convert + // has no verified small-batch GEMM kernel for these shapes (l4_lm drifts + // at small K). Fall back to CPU; decode (ne1==1) stays on the flat GEMV. + if (op->src[0]->type == GGML_TYPE_Q6_K && op->ne[1] > 1 && op->src[0]->ne[1] % 128 != 0) { + return false; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q8_0) { return op->src[1]->type == GGML_TYPE_F32; From f7eda48a0f8b27f8c71c13f9d1976b4888c49236 Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Mon, 6 Jul 2026 17:55:31 -0700 Subject: [PATCH 2/3] opencl: reserve alignment slack for the SOA subbuffer carve in alloc size set_tensor carves quantized weights into per-component subbuffers (d/q, ql/qh/s/d, ...) whose origins are each rounded up to the device base address alignment. When a component's size is not a multiple of the alignment, the carve extends past ggml_nbytes(tensor) and the last subbuffer overlaps the next tensor in the pool -- e.g. q6_K [1536, 49155]: size_s = 49155*96 ends 32 bytes past a 128-byte boundary, so the d subbuffer ends 96 bytes past the tensor's allocation, and whichever of the two neighboring tensors is uploaded last silently corrupts the other (here: the last vocab rows' block scales). This affects any quant type whose component sizes can be misaligned, on any shape with ne01 not a multiple of the alignment granularity; standard power-of-two dims are unaffected. Implement get_alloc_size for the OpenCL buffer type and reserve the worst-case carve slack (4 aligned gaps; 5 components max, q5_K) for quantized tensors. Costs at most 512 bytes per quantized tensor at the observed 128-byte alignment. --- ggml/src/ggml-opencl/ggml-opencl.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 3abe274f4999..9d58fcbe88fe 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -9798,12 +9798,30 @@ static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_buffer UNUSED(buft); } +static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + size_t size = ggml_nbytes(tensor); +#ifdef GGML_OPENCL_SOA_Q + // set_tensor carves quantized weights into per-component subbuffers (d/q, + // ql/qh/s/d, ...) whose origins are each rounded up to the device base + // alignment. When a component's size is not a multiple of the alignment + // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte + // alignment), the aligned carve extends past ggml_nbytes and the last + // subbuffer would overlap the next tensor in the pool. Reserve the worst-case + // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps. + if (ggml_is_quantized(tensor->type)) { + ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context; + size += 4 * dev_ctx->backend_ctx->alignment; + } +#endif // GGML_OPENCL_SOA_Q + return size; +} + static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = { /* .get_name = */ ggml_backend_opencl_buffer_type_get_name, /* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer, /* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment, /* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size, - /* .get_alloc_size = */ NULL, + /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size, /* .is_host = */ NULL, }; From 0f2bce237b692f598a5aa7cec6601ce7fe3d9444 Mon Sep 17 00:00:00 2001 From: Li He Date: Wed, 8 Jul 2026 11:30:27 -0700 Subject: [PATCH 3/3] opencl: use lm based q6_k mm when ne1 is not multiple of 128 --- ggml/src/ggml-opencl/ggml-opencl.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 9d58fcbe88fe..38600f31a7e9 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -6242,8 +6242,14 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c threshold_ne0 = 128; threshold_ne1 = 128; } - return tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && + bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 && tensor->ne[2] == 1 && tensor->ne[3] == 1; + + // q6_K adreno kernels requires ne1 is multiple of 128 + if (tensor->type == GGML_TYPE_Q6_K) { + return threashold_ok && tensor->ne[1] % 128 == 0; + } + return threashold_ok; } inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { @@ -6492,15 +6498,6 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || op->src[0]->type == GGML_TYPE_Q6_K) { -#ifdef GGML_OPENCL_USE_ADRENO_KERNELS - // q6_K GEMM (ne1>1) for weights routed off the noshuffle path because - // ne01 % 128 != 0 (see use_flat_gemv_for_large_m_q6_K): the flat convert - // has no verified small-batch GEMM kernel for these shapes (l4_lm drifts - // at small K). Fall back to CPU; decode (ne1==1) stays on the flat GEMV. - if (op->src[0]->type == GGML_TYPE_Q6_K && op->ne[1] > 1 && op->src[0]->ne[1] % 128 != 0) { - return false; - } -#endif // GGML_OPENCL_USE_ADRENO_KERNELS return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q8_0) { return op->src[1]->type == GGML_TYPE_F32;