Skip to content

Commit 842ae3f

Browse files
TheTomclaude
authored andcommitted
perf: fp16 centroid LUT — decode +6-14% at long context (TheTom#33)
Half-precision centroid table in vec flash attention dequant. Reduces constant cache pressure at high access volumes. Decode improvements: Short: 75.3 → 77.2 (+2.5%) 8K: 59.2 → 67.3 (+13.7%) 48K (Mario PDF): 36.7 → 39.0 (+6.3%) PPL: unchanged (6.211) Prefill: no regression Fixes TheTom#33 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-Authored-By: tturney@psyguard.ai
1 parent 613b455 commit 842ae3f

1 file changed

Lines changed: 13 additions & 8 deletions

File tree

ggml/src/ggml-metal/ggml-metal.metal

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -688,21 +688,26 @@ void dequantize_turbo3_0(device const block_turbo3_0 * xb, short il, thread type
688688
reg = (type4x4) reg_f;
689689
}
690690

691+
// Half-precision centroid LUT for vec path — reduces constant cache pressure at long context
692+
constant half turbo_centroids_3bit_h[8] = {
693+
-0.190685h, -0.117832h, -0.065717h, -0.021460h,
694+
0.021460h, 0.065717h, 0.117832h, 0.190685h
695+
};
696+
691697
// Vec: 4 elements per call (il ∈ {0..7}), returns type4
692698
template <typename type4>
693699
void dequantize_turbo3_0_t4(device const block_turbo3_0 * xb, short il, thread type4 & reg) {
694-
const float norm = float(xb->norm);
695-
// Codex-verified indexing: qbyte = qs[il], sbyte = signs[il>>1], sbase = (il&1)<<2
700+
const half nh = xb->norm;
696701
const uint8_t qb = xb->qs[il];
697702
const uint8_t sb = xb->signs[il >> 1];
698703
const int sshift = (il & 1) << 2;
699704

700-
reg = type4(
701-
turbo_centroids_3bit[(qb & 0x03) | (((sb >> (sshift + 0)) & 1) << 2)] * norm,
702-
turbo_centroids_3bit[((qb >> 2) & 0x03) | (((sb >> (sshift + 1)) & 1) << 2)] * norm,
703-
turbo_centroids_3bit[((qb >> 4) & 0x03) | (((sb >> (sshift + 2)) & 1) << 2)] * norm,
704-
turbo_centroids_3bit[((qb >> 6) & 0x03) | (((sb >> (sshift + 3)) & 1) << 2)] * norm
705-
);
705+
reg = type4(float4(half4(
706+
turbo_centroids_3bit_h[(qb & 0x03) | (((sb >> (sshift + 0)) & 1) << 2)] * nh,
707+
turbo_centroids_3bit_h[((qb >> 2) & 0x03) | (((sb >> (sshift + 1)) & 1) << 2)] * nh,
708+
turbo_centroids_3bit_h[((qb >> 4) & 0x03) | (((sb >> (sshift + 2)) & 1) << 2)] * nh,
709+
turbo_centroids_3bit_h[((qb >> 6) & 0x03) | (((sb >> (sshift + 3)) & 1) << 2)] * nh
710+
)));
706711
}
707712

708713
// ----- turbo4 dequantize with per-thread block cache -----

0 commit comments

Comments
 (0)