From c00217e918139f86b336f6089b2a6bde31e869fa Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Mon, 27 Apr 2026 14:10:01 +0800 Subject: [PATCH 1/8] ggml-cpu: fuse RMS_NORM + MUL on CPU backend Add a fused rms_norm + mul kernel that computes the output in a single pass, avoiding materialization of the intermediate rms_norm result. The graph compute loop detects the pattern and dispatches to the fused path. fusion can be disabled via the GGML_CPU_DISABLE_FUSION environment variable. --- ggml/src/ggml-cpu/ggml-cpu.c | 54 ++++++++++++++++++++++++++++++- ggml/src/ggml-cpu/ops.cpp | 62 ++++++++++++++++++++++++++++++++++++ ggml/src/ggml-cpu/ops.h | 1 + 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 2d6cc1fcd464..3fad4b49188e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2965,6 +2965,53 @@ struct ggml_cplan ggml_graph_plan( return cplan; } + +// Read env GGML_CPU_DISABLE_FUSION once and cache it +static bool get_disable_fusion(void) { + static bool value = false; + static bool initialized = false; + if (!initialized) { + value = (getenv("GGML_CPU_DISABLE_FUSION") != NULL); + initialized = true; + } + return value; +} + +// Try to fuse the current node with subsequent nodes for better performance. +// Returns true if fusion was applied. +static bool ggml_cpu_try_fuse_ops( + const struct ggml_cgraph * cgraph, + const int node_n, + const struct ggml_compute_params * params, + const struct ggml_cplan * cplan) { + if (get_disable_fusion() || cplan->use_ref) { + return false; + } + + struct ggml_tensor * node = cgraph->nodes[node_n]; + + if (node->op == GGML_OP_RMS_NORM) { + // RMS_NORM + MUL fusion + const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; + if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) { + struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1]; + const struct ggml_tensor * mul_w = (mul_node->src[0] == node) + ? mul_node->src[1] : mul_node->src[0]; + if (node->src[0]->type == GGML_TYPE_F32 && + mul_node->type == GGML_TYPE_F32 && + mul_w->type == GGML_TYPE_F32 && + mul_w->ne[0] == node->ne[0] && + mul_w->nb[0] == sizeof(float)) { + + ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); + return true; + } + } + } + + return false; +} + static thread_ret_t ggml_graph_compute_thread(void * data) { struct ggml_compute_state * state = (struct ggml_compute_state *) data; struct ggml_threadpool * tp = state->threadpool; @@ -3001,7 +3048,12 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { continue; } - ggml_compute_forward(¶ms, node); + // Try fused ops, fall back to normal compute + if (ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan)) { + node_n++; + } else { + ggml_compute_forward(¶ms, node); + } if (state->ith == 0 && cplan->abort_callback && cplan->abort_callback(cplan->abort_callback_data)) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 211f1ba1b2fa..43f76aa69609 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3782,6 +3782,68 @@ void ggml_compute_forward_rms_norm( } } +// Fused RMS_NORM + MUL: computes dst = rms_norm(src0) * src1 in a single pass. +// This avoids materializing the intermediate rms_norm result in memory. +void ggml_compute_forward_rms_norm_mul_fused( + const ggml_compute_params * params, + ggml_tensor * rms_norm_dst, + ggml_tensor * mul_dst) { + + // src0 = rms_norm input, src1 = mul weight, dst = mul output + const ggml_tensor * src0 = rms_norm_dst->src[0]; + const ggml_tensor * src1 = (mul_dst->src[0] == rms_norm_dst) ? mul_dst->src[1] : mul_dst->src[0]; + ggml_tensor * dst = mul_dst; + GGML_ASSERT(mul_dst->src[0] == rms_norm_dst || mul_dst->src[1] == rms_norm_dst); + + GGML_ASSERT(ggml_are_same_shape(src0, dst)); + GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT(src1->nb[0] == sizeof(float)); + + const int ith = params->ith; + const int nth = params->nth; + + GGML_TENSOR_BINARY_OP_LOCALS + + GGML_ASSERT(ne10 == ne00); // no column broadcasting on the weight + + float eps; + memcpy(&eps, rms_norm_dst->op_params, sizeof(float)); + GGML_ASSERT(eps >= 0.0f); + for (int64_t i03 = 0; i03 < ne03; i03++) { + for (int64_t i02 = 0; i02 < ne02; i02++) { + for (int64_t i01 = ith; i01 < ne01; i01 += nth) { + const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); + + ggml_float sum = 0.0; + // worth switching to explicit SIMD? + for (int64_t i00 = 0; i00 < ne00; i00++) { + sum += (ggml_float)(x[i00] * x[i00]); + } + + const float mean = sum/ne00; + const float scale = 1.0f / sqrtf(mean + eps); + + assert(scale > 0.0f); + + const int64_t i11 = i01 % ne11; + const int64_t i12 = i02 % ne12; + const int64_t i13 = i03 % ne13; + const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13); + + float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + + // worth switching to explicit SIMD? + for (int64_t i00 = 0; i00 < ne00; i00++) { + y[i00] = x[i00] * scale * w[i00]; + } + } + } + } +} + static void ggml_compute_forward_rms_norm_back_f32( const ggml_compute_params * params, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 29efdeee37f2..ecd8dde14e98 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -44,6 +44,7 @@ void ggml_compute_forward_concat(const struct ggml_compute_params * params, stru void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * rms_norm_dst, struct ggml_tensor * mul_dst); void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); From 6f03632e23a2f3b98d4da06e4b9d1ca2b95f6e77 Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Tue, 28 Apr 2026 10:33:23 +0800 Subject: [PATCH 2/8] ggml-cpu : address review feedback on fusion path - inline get_disable_fusion() into ggml_cpu_try_fuse_ops - fix env check: only disable on "1"/"on"/"true" (case-insensitive) - return fused node count (int) instead of bool, for future N-op fusion - add TODO to move fusion detection into ggml_graph_plan --- ggml/src/ggml-cpu/ggml-cpu.c | 51 ++++++++++------ ggml/src/ggml-cpu/ops.cpp | 113 +++++++++++++++-------------------- 2 files changed, 81 insertions(+), 83 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 3fad4b49188e..b52e98d164ae 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -22,6 +22,7 @@ #endif #include +#include #include #include #include @@ -2966,26 +2967,36 @@ struct ggml_cplan ggml_graph_plan( } -// Read env GGML_CPU_DISABLE_FUSION once and cache it -static bool get_disable_fusion(void) { - static bool value = false; - static bool initialized = false; - if (!initialized) { - value = (getenv("GGML_CPU_DISABLE_FUSION") != NULL); - initialized = true; - } - return value; -} - // Try to fuse the current node with subsequent nodes for better performance. -// Returns true if fusion was applied. -static bool ggml_cpu_try_fuse_ops( +// Returns the number of nodes consumed by fusion (>=2), or 0 if no fusion was applied. +static int ggml_cpu_try_fuse_ops( const struct ggml_cgraph * cgraph, const int node_n, const struct ggml_compute_params * params, const struct ggml_cplan * cplan) { - if (get_disable_fusion() || cplan->use_ref) { - return false; + + // Read env GGML_CPU_DISABLE_FUSION once and cache it + static bool disable_fusion = false; + static bool disable_fusion_initialized = false; + if (!disable_fusion_initialized) { + const char * env = getenv("GGML_CPU_DISABLE_FUSION"); + if (env != NULL) { + // accept: "1", "on", "true" (case-insensitive) + char buf[8] = {0}; + size_t i = 0; + for (; env[i] != '\0' && i < sizeof(buf) - 1; ++i) { + buf[i] = (char) tolower((unsigned char) env[i]); + } + buf[i] = '\0'; + disable_fusion = (strcmp(buf, "1") == 0 || + strcmp(buf, "on") == 0 || + strcmp(buf, "true") == 0); + } + disable_fusion_initialized = true; + } + + if (disable_fusion || cplan->use_ref) { + return 0; } struct ggml_tensor * node = cgraph->nodes[node_n]; @@ -3004,12 +3015,12 @@ static bool ggml_cpu_try_fuse_ops( mul_w->nb[0] == sizeof(float)) { ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); - return true; + return 2; } } } - return false; + return 0; } static thread_ret_t ggml_graph_compute_thread(void * data) { @@ -3048,9 +3059,11 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { continue; } + // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time // Try fused ops, fall back to normal compute - if (ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan)) { - node_n++; + const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); + if (n_fused > 0) { + node_n += n_fused - 1; } else { ggml_compute_forward(¶ms, node); } diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 43f76aa69609..233d57730c19 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3713,11 +3713,28 @@ void ggml_compute_forward_norm( // ggml_compute_forward_group_rms_norm +// fusion kinds that can be combined with the rms_norm computation in a single pass. +// extend this enum when adding new fused variants (e.g. FUSE_ADD, FUSE_MUL_ADD, ...). +enum ggml_rms_norm_fuse_op { + GGML_RMS_NORM_FUSE_NONE, + GGML_RMS_NORM_FUSE_MUL, +}; + +template static void ggml_compute_forward_rms_norm_f32( const ggml_compute_params * params, - ggml_tensor * dst) { + ggml_tensor * rms_norm_dst, + ggml_tensor * fused_dst = nullptr) { - const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src0 = rms_norm_dst->src[0]; + const ggml_tensor * src1 = nullptr; + ggml_tensor * dst = rms_norm_dst; + + // ggml_compute_forward_rms_norm_mul_fused should do the graph fusion check. + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { + src1 = (fused_dst->src[0] == rms_norm_dst) ? fused_dst->src[1] : fused_dst->src[0]; + dst = fused_dst; + } GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -3726,11 +3743,10 @@ static void ggml_compute_forward_rms_norm_f32( const int ith = params->ith; const int nth = params->nth; - GGML_TENSOR_UNARY_OP_LOCALS + GGML_TENSOR_BINARY_OP_LOCALS float eps; - memcpy(&eps, dst->op_params, sizeof(float)); - + memcpy(&eps, rms_norm_dst->op_params, sizeof(float)); GGML_ASSERT(eps >= 0.0f); // TODO: optimize @@ -3740,25 +3756,32 @@ static void ggml_compute_forward_rms_norm_f32( const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); ggml_float sum = 0.0; + // worth switching to explicit SIMD? for (int64_t i00 = 0; i00 < ne00; i00++) { sum += (ggml_float)(x[i00] * x[i00]); } - const float mean = sum/ne00; - - float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - - memcpy(y, x, ne00 * sizeof(float)); - // for (int i00 = 0; i00 < ne00; i00++) { - // y[i00] = x[i00]; - // } - + const float mean = sum/ne00; const float scale = 1.0f/sqrtf(mean + eps); // if you hit this, likely you got an inf somewhere earlier assert(scale > 0.0f); - ggml_vec_scale_f32(ne00, y, scale); + float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { + const int64_t i11 = i01 % ne11; + const int64_t i12 = i02 % ne12; + const int64_t i13 = i03 % ne13; + const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13); + + for (int64_t i00 = 0; i00 < ne00; i00++) { + y[i00] = x[i00] * scale * w[i00]; + } + } else { + memcpy(y, x, ne00 * sizeof(float)); + ggml_vec_scale_f32(ne00, y, scale); + } } } } @@ -3773,7 +3796,7 @@ void ggml_compute_forward_rms_norm( switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, dst); + ggml_compute_forward_rms_norm_f32(params, dst); } break; default: { @@ -3789,58 +3812,20 @@ void ggml_compute_forward_rms_norm_mul_fused( ggml_tensor * rms_norm_dst, ggml_tensor * mul_dst) { - // src0 = rms_norm input, src1 = mul weight, dst = mul output - const ggml_tensor * src0 = rms_norm_dst->src[0]; - const ggml_tensor * src1 = (mul_dst->src[0] == rms_norm_dst) ? mul_dst->src[1] : mul_dst->src[0]; - ggml_tensor * dst = mul_dst; + GGML_ASSERT(mul_dst != nullptr); GGML_ASSERT(mul_dst->src[0] == rms_norm_dst || mul_dst->src[1] == rms_norm_dst); - GGML_ASSERT(ggml_are_same_shape(src0, dst)); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); - GGML_ASSERT(src0->nb[0] == sizeof(float)); - GGML_ASSERT(src1->nb[0] == sizeof(float)); - - const int ith = params->ith; - const int nth = params->nth; - - GGML_TENSOR_BINARY_OP_LOCALS - - GGML_ASSERT(ne10 == ne00); // no column broadcasting on the weight - - float eps; - memcpy(&eps, rms_norm_dst->op_params, sizeof(float)); - GGML_ASSERT(eps >= 0.0f); - for (int64_t i03 = 0; i03 < ne03; i03++) { - for (int64_t i02 = 0; i02 < ne02; i02++) { - for (int64_t i01 = ith; i01 < ne01; i01 += nth) { - const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03); - - ggml_float sum = 0.0; - // worth switching to explicit SIMD? - for (int64_t i00 = 0; i00 < ne00; i00++) { - sum += (ggml_float)(x[i00] * x[i00]); - } - - const float mean = sum/ne00; - const float scale = 1.0f / sqrtf(mean + eps); - - assert(scale > 0.0f); - - const int64_t i11 = i01 % ne11; - const int64_t i12 = i02 % ne12; - const int64_t i13 = i03 % ne13; - const float * w = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13); - - float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); + const ggml_tensor * src0 = rms_norm_dst->src[0]; - // worth switching to explicit SIMD? - for (int64_t i00 = 0; i00 < ne00; i00++) { - y[i00] = x[i00] * scale * w[i00]; - } + switch (src0->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_rms_norm_f32(params, rms_norm_dst, mul_dst); + } break; + default: + { + GGML_ABORT("fatal error"); } - } } } From c545efa05144964cdc83a8d9ffd4037b07fbe042 Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Tue, 28 Apr 2026 12:15:26 +0800 Subject: [PATCH 3/8] ggml-cpu : simplify fusion-disable env check with atoi --- ggml/src/ggml-cpu/ggml-cpu.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index b52e98d164ae..0fbc88533331 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -22,7 +22,6 @@ #endif #include -#include #include #include #include @@ -2980,18 +2979,7 @@ static int ggml_cpu_try_fuse_ops( static bool disable_fusion_initialized = false; if (!disable_fusion_initialized) { const char * env = getenv("GGML_CPU_DISABLE_FUSION"); - if (env != NULL) { - // accept: "1", "on", "true" (case-insensitive) - char buf[8] = {0}; - size_t i = 0; - for (; env[i] != '\0' && i < sizeof(buf) - 1; ++i) { - buf[i] = (char) tolower((unsigned char) env[i]); - } - buf[i] = '\0'; - disable_fusion = (strcmp(buf, "1") == 0 || - strcmp(buf, "on") == 0 || - strcmp(buf, "true") == 0); - } + disable_fusion = (env != NULL && atoi(env) == 1); disable_fusion_initialized = true; } From 7012a88be089c5169312d10a223956868a896bec Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Tue, 28 Apr 2026 15:05:44 +0800 Subject: [PATCH 4/8] ggml-cpu: remove redundant comments --- ggml/src/ggml-cpu/ggml-cpu.c | 1 - ggml/src/ggml-cpu/ops.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 0fbc88533331..1ca116e17036 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2974,7 +2974,6 @@ static int ggml_cpu_try_fuse_ops( const struct ggml_compute_params * params, const struct ggml_cplan * cplan) { - // Read env GGML_CPU_DISABLE_FUSION once and cache it static bool disable_fusion = false; static bool disable_fusion_initialized = false; if (!disable_fusion_initialized) { diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 233d57730c19..d4d80c9de880 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3730,7 +3730,6 @@ static void ggml_compute_forward_rms_norm_f32( const ggml_tensor * src1 = nullptr; ggml_tensor * dst = rms_norm_dst; - // ggml_compute_forward_rms_norm_mul_fused should do the graph fusion check. if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { src1 = (fused_dst->src[0] == rms_norm_dst) ? fused_dst->src[1] : fused_dst->src[0]; dst = fused_dst; From 036bbace8f8a85fb3cb95c2890603840f4298813 Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Wed, 29 Apr 2026 17:28:54 +0800 Subject: [PATCH 5/8] ggml-cpu : address review comments --- ggml/src/ggml-cpu/ggml-cpu.c | 17 ++++++++--------- ggml/src/ggml-cpu/ops.h | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 1ca116e17036..4989dc567199 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2968,21 +2968,15 @@ struct ggml_cplan ggml_graph_plan( // Try to fuse the current node with subsequent nodes for better performance. // Returns the number of nodes consumed by fusion (>=2), or 0 if no fusion was applied. +static bool ggml_cpu_disable_fusion = false; // initialized once in ggml_cpu_init(), read-only afterwards + static int ggml_cpu_try_fuse_ops( const struct ggml_cgraph * cgraph, const int node_n, const struct ggml_compute_params * params, const struct ggml_cplan * cplan) { - static bool disable_fusion = false; - static bool disable_fusion_initialized = false; - if (!disable_fusion_initialized) { - const char * env = getenv("GGML_CPU_DISABLE_FUSION"); - disable_fusion = (env != NULL && atoi(env) == 1); - disable_fusion_initialized = true; - } - - if (disable_fusion || cplan->use_ref) { + if (ggml_cpu_disable_fusion || cplan->use_ref) { return 0; } @@ -3815,6 +3809,11 @@ void ggml_cpu_init(void) { ggml_init_riscv_arch_features(); #endif + { + const char * env = getenv("GGML_CPU_DISABLE_FUSION"); + ggml_cpu_disable_fusion = (env != NULL && atoi(env) == 1); + } + is_first_call = false; } diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index ecd8dde14e98..7398e5618948 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -44,7 +44,7 @@ void ggml_compute_forward_concat(const struct ggml_compute_params * params, stru void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); -void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * rms_norm_dst, struct ggml_tensor * mul_dst); +void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * dst_rms_norm, struct ggml_tensor * dst_mul); void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_group_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_l2_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst); From ada950de482884a7d6481bcbb5484cdb304d6623 Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Thu, 30 Apr 2026 13:52:44 +0800 Subject: [PATCH 6/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sigbjørn Skjæret --- ggml/src/ggml-cpu/ggml-cpu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 4989dc567199..91d7b7b4e872 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2967,7 +2967,7 @@ struct ggml_cplan ggml_graph_plan( // Try to fuse the current node with subsequent nodes for better performance. -// Returns the number of nodes consumed by fusion (>=2), or 0 if no fusion was applied. +// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied. static bool ggml_cpu_disable_fusion = false; // initialized once in ggml_cpu_init(), read-only afterwards static int ggml_cpu_try_fuse_ops( @@ -2996,7 +2996,7 @@ static int ggml_cpu_try_fuse_ops( mul_w->nb[0] == sizeof(float)) { ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); - return 2; + return 1; } } } @@ -3044,7 +3044,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { // Try fused ops, fall back to normal compute const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); if (n_fused > 0) { - node_n += n_fused - 1; + node_n += n_fused; } else { ggml_compute_forward(¶ms, node); } From d2c7072721ff4d70cb3fb3642da7258b97d548bd Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Mon, 4 May 2026 15:09:14 +0800 Subject: [PATCH 7/8] ggml-cpu : address review comments - normalize to LF (\n) in ggml-cpu.c - rename variables in ops.cpp from xxx_dst to dst_xxx --- ggml/src/ggml-cpu/ggml-cpu.c | 6 +++--- ggml/src/ggml-cpu/ops.cpp | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 91d7b7b4e872..8b7acafdaa81 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2967,7 +2967,7 @@ struct ggml_cplan ggml_graph_plan( // Try to fuse the current node with subsequent nodes for better performance. -// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied. +// Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied. static bool ggml_cpu_disable_fusion = false; // initialized once in ggml_cpu_init(), read-only afterwards static int ggml_cpu_try_fuse_ops( @@ -2996,7 +2996,7 @@ static int ggml_cpu_try_fuse_ops( mul_w->nb[0] == sizeof(float)) { ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); - return 1; + return 1; } } } @@ -3044,7 +3044,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { // Try fused ops, fall back to normal compute const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); if (n_fused > 0) { - node_n += n_fused; + node_n += n_fused; } else { ggml_compute_forward(¶ms, node); } diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index d4d80c9de880..e8bc735c9a83 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3723,16 +3723,16 @@ enum ggml_rms_norm_fuse_op { template static void ggml_compute_forward_rms_norm_f32( const ggml_compute_params * params, - ggml_tensor * rms_norm_dst, - ggml_tensor * fused_dst = nullptr) { + ggml_tensor * dst_rms_norm, + ggml_tensor * dst_fused = nullptr) { - const ggml_tensor * src0 = rms_norm_dst->src[0]; + const ggml_tensor * src0 = dst_rms_norm->src[0]; const ggml_tensor * src1 = nullptr; - ggml_tensor * dst = rms_norm_dst; + ggml_tensor * dst = dst_rms_norm; if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { - src1 = (fused_dst->src[0] == rms_norm_dst) ? fused_dst->src[1] : fused_dst->src[0]; - dst = fused_dst; + src1 = (dst_fused->src[0] == dst_rms_norm) ? dst_fused->src[1] : dst_fused->src[0]; + dst = dst_fused; } GGML_ASSERT(ggml_are_same_shape(src0, dst)); @@ -3745,7 +3745,7 @@ static void ggml_compute_forward_rms_norm_f32( GGML_TENSOR_BINARY_OP_LOCALS float eps; - memcpy(&eps, rms_norm_dst->op_params, sizeof(float)); + memcpy(&eps, dst_rms_norm->op_params, sizeof(float)); GGML_ASSERT(eps >= 0.0f); // TODO: optimize @@ -3808,18 +3808,18 @@ void ggml_compute_forward_rms_norm( // This avoids materializing the intermediate rms_norm result in memory. void ggml_compute_forward_rms_norm_mul_fused( const ggml_compute_params * params, - ggml_tensor * rms_norm_dst, - ggml_tensor * mul_dst) { + ggml_tensor * dst_rms_norm, + ggml_tensor * dst_mul) { - GGML_ASSERT(mul_dst != nullptr); - GGML_ASSERT(mul_dst->src[0] == rms_norm_dst || mul_dst->src[1] == rms_norm_dst); + GGML_ASSERT(dst_mul != nullptr); + GGML_ASSERT(dst_mul->src[0] == dst_rms_norm || dst_mul->src[1] == dst_rms_norm); - const ggml_tensor * src0 = rms_norm_dst->src[0]; + const ggml_tensor * src0 = dst_rms_norm->src[0]; switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, rms_norm_dst, mul_dst); + ggml_compute_forward_rms_norm_f32(params, dst_rms_norm, dst_mul); } break; default: { From 41eb57a03505aea5acd37b5553759cb4fdfd0ea7 Mon Sep 17 00:00:00 2001 From: zzzzwc Date: Tue, 5 May 2026 16:30:13 +0800 Subject: [PATCH 8/8] ggml-cpu : address review comments - rename GGML_RMS_NORM_FUSE_XXX to GGML_RMS_NORM_FUSE_OP_xxx --- ggml/src/ggml-cpu/ops.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index e8bc735c9a83..6bc8dc150ce8 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -3716,8 +3716,8 @@ void ggml_compute_forward_norm( // fusion kinds that can be combined with the rms_norm computation in a single pass. // extend this enum when adding new fused variants (e.g. FUSE_ADD, FUSE_MUL_ADD, ...). enum ggml_rms_norm_fuse_op { - GGML_RMS_NORM_FUSE_NONE, - GGML_RMS_NORM_FUSE_MUL, + GGML_RMS_NORM_FUSE_OP_NONE, + GGML_RMS_NORM_FUSE_OP_MUL, }; template @@ -3730,7 +3730,7 @@ static void ggml_compute_forward_rms_norm_f32( const ggml_tensor * src1 = nullptr; ggml_tensor * dst = dst_rms_norm; - if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) { src1 = (dst_fused->src[0] == dst_rms_norm) ? dst_fused->src[1] : dst_fused->src[0]; dst = dst_fused; } @@ -3768,7 +3768,7 @@ static void ggml_compute_forward_rms_norm_f32( float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); - if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_MUL) { + if constexpr (FUSE_OP == GGML_RMS_NORM_FUSE_OP_MUL) { const int64_t i11 = i01 % ne11; const int64_t i12 = i02 % ne12; const int64_t i13 = i03 % ne13; @@ -3795,7 +3795,7 @@ void ggml_compute_forward_rms_norm( switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, dst); + ggml_compute_forward_rms_norm_f32(params, dst); } break; default: { @@ -3819,7 +3819,7 @@ void ggml_compute_forward_rms_norm_mul_fused( switch (src0->type) { case GGML_TYPE_F32: { - ggml_compute_forward_rms_norm_f32(params, dst_rms_norm, dst_mul); + ggml_compute_forward_rms_norm_f32(params, dst_rms_norm, dst_mul); } break; default: {