Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ggml/src/ggml-cpu/ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4894,6 +4894,8 @@ void ggml_compute_forward_get_rows(
//}
}

extern "C" GGML_API int turbo3_cpu_wht_group_size;

template<typename idx_t>
static void ggml_compute_forward_set_rows_f32(
const ggml_compute_params * params,
Expand Down Expand Up @@ -4928,7 +4930,6 @@ static void ggml_compute_forward_set_rows_f32(

// For turbo types: communicate WHT group size to the quantize function via global
if (dst->type == GGML_TYPE_TURBO3_0 || dst->type == GGML_TYPE_TURBO4_0 || dst->type == GGML_TYPE_TURBO2_0) {
extern int turbo3_cpu_wht_group_size;
int gs = 0;
memcpy(&gs, dst->op_params, sizeof(int));
turbo3_cpu_wht_group_size = (gs == 64 || gs == 128) ? gs : 0;
Expand Down
8 changes: 5 additions & 3 deletions ggml/src/ggml-cuda/turbo-innerq.cuh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#pragma once

#include "ggml.h"

// TurboQuant InnerQ per-channel equalization — cross-TU shared state
// The host-side state lives in turbo-innerq.cu; device-side state is per-TU
// in turbo-quant.cuh (only set-rows.cu needs device access).
Expand All @@ -8,14 +10,14 @@

// Host-side shared state (defined in turbo-innerq.cu)
extern bool g_innerq_finalized;
extern float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];
GGML_API float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];

// Called from set-rows.cu after InnerQ finalization to publish scale_inv
void turbo_innerq_publish(const float * scale_inv, int group_size);

// Called from llama-kv-cache.cpp (or equivalent) to check if tensor needs update
// Returns true if there are new scale_inv values to upload
bool turbo_innerq_needs_tensor_update(void);
GGML_API bool turbo_innerq_needs_tensor_update(void);

// Called after tensor update to clear the flag
void turbo_innerq_mark_tensor_updated(void);
GGML_API void turbo_innerq_mark_tensor_updated(void);
8 changes: 5 additions & 3 deletions ggml/src/ggml-turbo-quant.c
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,17 @@
#include "ggml-common.h"
#include "ggml-impl.h"

#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
#define _USE_MATH_DEFINES
#endif

#include <math.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>

/* Global: WHT group size for CPU quantize path (set by CPU SET_ROWS handler) */
int turbo3_cpu_wht_group_size = 0;
GGML_API int turbo3_cpu_wht_group_size = 0;

/* ---------- constants ---------- */

Expand Down Expand Up @@ -246,7 +250,6 @@ void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * G

// Read WHT group size from global (set by CPU SET_ROWS handler before each call).
// Fallback: 128 if row is 128-aligned, else 64.
extern int turbo3_cpu_wht_group_size;
int group_size = turbo3_cpu_wht_group_size;
if (group_size != 64 && group_size != 128) {
group_size = (k % 128 == 0) ? 128 : 64;
Expand Down Expand Up @@ -341,7 +344,6 @@ size_t quantize_turbo3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT d
void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k) {
assert(k % QK_TURBO2 == 0);

extern int turbo3_cpu_wht_group_size;
int group_size = turbo3_cpu_wht_group_size;
if (group_size != 64 && group_size != 128) {
group_size = (k % 128 == 0) ? 128 : 64;
Expand Down
8 changes: 4 additions & 4 deletions src/llama-kv-cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@
#endif

#ifdef GGML_USE_CUDA
extern bool g_innerq_finalized;
extern float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];
extern bool turbo_innerq_needs_tensor_update(void);
extern void turbo_innerq_mark_tensor_updated(void);
GGML_API bool g_innerq_finalized;
GGML_API float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];
GGML_API bool turbo_innerq_needs_tensor_update(void);
GGML_API void turbo_innerq_mark_tensor_updated(void);
#else
static bool g_innerq_finalized = false;
static float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS] = {};
Expand Down