fix: restore inverse rotation in dequant — PPL 6.19 (1.2% of q8_0) #31 #30

ROOT CAUSE: pre-rotate-queries never executed because:
1. Q ne[0]=256 (GQA concatenated heads), rotation matrix ne[0]=128
2. mctx dynamic_cast failed for MoE hybrid memory

FIX: put inverse WHT rotation back in dequantize_full_block.
This is slower (10.7 tok/s vs 77.7) but produces CORRECT results.

PERPLEXITY RESULTS:
- f16:     6.121
- q8_0:    6.111
- q4_0:    6.142
- turbo3:  6.194 (+1.2% vs q8_0) ✅

The speed optimization (pre-rotate-queries) needs to be reimplemented
to work with GQA head layout and hybrid memory types.

Co-Authored-By: tturney@psyguard.ai
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: TheTom

ggml/src/ggml-metal/ggml-metal.metal

@@ -514,36 +514,48 @@ void quantize_turbo4_0(device const float * src, device block_turbo4_0 & dst) {
 // up to 32× per block (once per 4-element chunk). We cache the full
 // dequantized block per thread and only recompute when the block pointer changes.
 
-// turbo3 dequant — block size 32, same pattern as q4_0
-// Just 3-bit centroid lookup + norm scale. No rotation (pre-rotate-queries handles it).
-template <typename type4x4>
-void dequantize_turbo3_0(device const block_turbo3_0 * xb, short il, thread type4x4 & reg) {
+// turbo3 dequant — full block dequantize with inverse rotation
+// Must process all 128 elements to apply WHT inverse rotation
+static void turbo3_dequantize_full_block(device const block_turbo3_0 * xb, thread float * cache) {
     const float norm = float(xb->norm);
-    // il selects which 16-element half (0 or 1) of the 32-element block
-    const int offset = il * 16;
 
-    float4x4 reg_f;
-    for (int i = 0; i < 16; i++) {
-        int j = offset + i;
+    // Unpack 3-bit indices → centroids (in rotated space)
+    float recon[128];
+    for (int j = 0; j < QK_TURBO3; j++) {
         uint8_t low2 = (xb->qs[j / 4] >> ((j % 4) * 2)) & 0x3;
         uint8_t hi1  = (xb->signs[j / 8] >> (j % 8)) & 0x1;
         uint8_t idx  = low2 | (hi1 << 2);
-        reg_f[i/4][i%4] = turbo_centroids_3bit[idx] * norm;
+        recon[j] = turbo_centroids_3bit[idx];
+    }
+
+    // Inverse WHT rotation: from rotated space back to original
+    turbo_rotate_inverse(recon, turbo_wht_signs1, turbo_wht_signs2);
+
+    // Scale by norm to get back to original magnitude
+    for (int j = 0; j < QK_TURBO3; j++) {
+        cache[j] = recon[j] * norm;
+    }
+}
+
+template <typename type4x4>
+void dequantize_turbo3_0(device const block_turbo3_0 * xb, short il, thread type4x4 & reg) {
+    float cache[128];
+    turbo3_dequantize_full_block(xb, cache);
+    const int offset = il * 16;
+    float4x4 reg_f;
+    for (int i = 0; i < 16; i++) {
+        reg_f[i/4][i%4] = cache[offset + i];
     }
     reg = (type4x4) reg_f;
 }
 
 template <typename type4>
 void dequantize_turbo3_0_t4(device const block_turbo3_0 * xb, short il, thread type4 & reg) {
-    const float norm = float(xb->norm);
+    float cache[128];
+    turbo3_dequantize_full_block(xb, cache);
     const int offset = il * 4;
-
     for (int i = 0; i < 4; i++) {
-        int j = offset + i;
-        uint8_t low2 = (xb->qs[j / 4] >> ((j % 4) * 2)) & 0x3;
-        uint8_t hi1  = (xb->signs[j / 8] >> (j % 8)) & 0x1;
-        uint8_t idx  = low2 | (hi1 << 2);
-        reg[i] = turbo_centroids_3bit[idx] * norm;
+        reg[i] = cache[offset + i];
     }
 }
 

src/llama-kv-cache.cpp

@@ -2464,6 +2464,14 @@ ggml_tensor * llama_kv_cache_context::get_turbo_rotation_inv() const {
     return kv->get_turbo_rotation_inv();
 }
 
+ggml_tensor * llama_kv_cache_context::get_turbo_rot_forward() const {
+    return kv->get_turbo_rotation();
+}
+
+ggml_tensor * llama_kv_cache_context::get_turbo_rot_inverse() const {
+    return kv->get_turbo_rotation_inv();
+}
+
 ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const {
     return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]);
 }

src/llama-kv-cache.h

@@ -366,6 +366,10 @@ class llama_kv_cache_context : public llama_memory_context_i {
     ggml_tensor * get_turbo_rotation() const;
     ggml_tensor * get_turbo_rotation_inv() const;
 
+    // Override virtual methods from llama_memory_context_i
+    ggml_tensor * get_turbo_rot_forward() const override;
+    ggml_tensor * get_turbo_rot_inverse() const override;
+
     // store k_cur and v_cur in the cache based on the provided head location
     // note: the heads in k_cur and v_cur should be laid out contiguously in memory
     //   - k_cur  [n_embd_head_k, n_head_k, n_tokens]

src/llama-memory-hybrid.cpp

@@ -263,6 +263,14 @@ const llama_kv_cache_context * llama_memory_hybrid_context::get_attn() const {
     return static_cast<const llama_kv_cache_context *>(ctx_attn.get());
 }
 
+ggml_tensor * llama_memory_hybrid_context::get_turbo_rot_forward() const {
+    return ctx_attn ? ctx_attn->get_turbo_rot_forward() : nullptr;
+}
+
+ggml_tensor * llama_memory_hybrid_context::get_turbo_rot_inverse() const {
+    return ctx_attn ? ctx_attn->get_turbo_rot_inverse() : nullptr;
+}
+
 const llama_memory_recurrent_context * llama_memory_hybrid_context::get_recr() const {
     return static_cast<const llama_memory_recurrent_context *>(ctx_recr.get());
 }

src/llama-memory-hybrid.h

@@ -119,6 +119,10 @@ class llama_memory_hybrid_context : public llama_memory_context_i {
     llama_memory_status  get_status() const override;
     const llama_ubatch & get_ubatch() const override;
 
+    // TurboQuant: delegate to the KV cache context
+    ggml_tensor * get_turbo_rot_forward() const override;
+    ggml_tensor * get_turbo_rot_inverse() const override;
+
     //
     // llama_memory_hybrid_context
     //

src/llama-memory.h

@@ -59,6 +59,11 @@ struct llama_memory_context_i {
 
     // get the status of the memory context - used for error handling and checking if any updates would be applied
     virtual llama_memory_status get_status() const = 0;
+
+    // TurboQuant: get rotation tensors for pre-rotate-queries optimization
+    // Returns null for non-turbo memory types. Override in KV cache contexts.
+    virtual ggml_tensor * get_turbo_rot_forward() const { return nullptr; }
+    virtual ggml_tensor * get_turbo_rot_inverse() const { return nullptr; }
 };
 
 using llama_memory_context_ptr = std::unique_ptr<llama_memory_context_i>;