From ece2adf3148d3702d1cfd54a4e3cafc5f0571e8d Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sat, 16 May 2026 05:42:28 +0000 Subject: [PATCH 1/3] Option to use re-quantized output tensor for MTP --- common/common.cpp | 20 +++ common/common.h | 2 + examples/quantize/quantize.cpp | 9 +- ggml/src/iqk/iqk_quantize.cpp | 12 +- include/llama.h | 3 + src/graphs/build_qwen35.cpp | 4 +- src/llama-load-tensors.cpp | 20 ++- src/llama-model-loader.h | 4 + src/llama-model.h | 3 + src/llama-quantize.cpp | 224 ++++++++++++++++++++++++--------- src/llama.cpp | 86 ++++++++++++- 11 files changed, 315 insertions(+), 72 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index dccfe1db98..2e36b2882d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1727,6 +1727,11 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa } return true; } + if (arg == "--mtp-requantized-output-tensor" || arg == "-mtprot") { + CHECK_ARG + params.extra_output_type = argv[i]; + return true; + } if (arg == "-ctkd" || arg == "--cache-type-k-draft") { params.speculative.cache_type_k = argv[++i]; return true; @@ -3028,6 +3033,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param options.push_back({ "*", "-ctv-last, --cache-type-k-last TYPE,N", "KV cache data type for the last N layers of K (default: %s,-1)", params.type_k_last.c_str() }); options.push_back({ "*", "-ctv-first, --cache-type-v-first TYPE,N", "KV cache data type for the first N layers of V (default: %s,-1)", params.type_v_first.c_str() }); options.push_back({ "*", "-ctk-last, --cache-type-v-last TYPE,N", "KV cache data type for the last N layers of V (default: %s,-1)", params.type_v_last.c_str() }); + options.push_back({ "*", "-mtprot, --mtp-requantize-output-tensor type", "Use output requantized to type for MTP (default: %s)", params.extra_output_type.c_str() }); options.push_back({ "*", "-ctkd, --cache-type-k-draft TYPE", "KV cache data type for K for the draft model" }); options.push_back({ "*", "-ctvd, --cache-type-v-draft TYPE", "KV cache data type for V for the draft model" }); @@ -3926,6 +3932,17 @@ static std::pair get_batch_ubatch(const gpt_params & params) { return {n_batch, n_ubatch}; } +static ggml_type parse_ggml_type(const char * arg) { + for (int j = 0; j < GGML_TYPE_COUNT; ++j) { + auto type = ggml_type(j); + const auto * name = ggml_type_name(type); + if (name && strcmp(arg, name) == 0) { + return type; + } + } + return GGML_TYPE_COUNT; +} + struct llama_model_params common_model_params_to_llama(const gpt_params & params) { auto mparams = llama_model_default_params(); mparams.devices = params.devices.c_str(); @@ -3948,6 +3965,9 @@ struct llama_model_params common_model_params_to_llama(const gpt_params & params mparams.type_k_last = kv_cache_type_from_str(params.type_k_last ); mparams.type_v_first = kv_cache_type_from_str(params.type_v_first); mparams.type_v_last = kv_cache_type_from_str(params.type_v_last ); + if (!params.extra_output_type.empty()) { + mparams.extra_output_type = parse_ggml_type(params.extra_output_type.c_str()); + } mparams.n_k_first = params.n_k_first; mparams.n_k_last = params.n_k_last; mparams.n_v_first = params.n_v_first; diff --git a/common/common.h b/common/common.h index 2ea38928fe..0dbc29b7db 100644 --- a/common/common.h +++ b/common/common.h @@ -443,6 +443,8 @@ struct gpt_params { int32_t n_v_first = -1; int32_t n_v_last = -1; + std::string extra_output_type = ""; + // multimodal models (see examples/mtmd) common_params_model mmproj; bool mmproj_use_gpu = true; // use GPU for multimodal model diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 83f797220b..3734a67d6e 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -151,7 +151,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); @@ -163,6 +163,7 @@ static void usage(const char * executable) { printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor.\n"); printf(" --token-embedding-type ggml_type: use this ggml_type for the token_embd.weight tensor.\n\n"); + printf(" --extra-output-tensor ggml_type: requantize and add output tensor of that type.\n"); printf(" --ffn-gate-inp-type ggml_type: use this ggml_type for the ffn_gate_inp tensors.\n\n"); printf(" --custom-q regex1=type1,regex2=type2...: use this to specify custom quantization type rules.\n\n"); printf(" --repack Repack all tensors to the corresponding _r4/8 variant if available.\n\n"); @@ -383,6 +384,12 @@ int main(int argc, char ** argv) { } else { usage(argv[0]); } + } else if (strcmp(argv[arg_idx], "--extra-output-tensor") == 0) { + if (arg_idx < argc-1) { + params.extra_output_type = parse_ggml_type(argv[++arg_idx]); + } else { + usage(argv[0]); + } } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) { if (arg_idx < argc-1) { params.token_embedding_type = parse_ggml_type(argv[++arg_idx]); diff --git a/ggml/src/iqk/iqk_quantize.cpp b/ggml/src/iqk/iqk_quantize.cpp index 7f183468e1..fd12f6b594 100644 --- a/ggml/src/iqk/iqk_quantize.cpp +++ b/ggml/src/iqk/iqk_quantize.cpp @@ -260,12 +260,16 @@ void iqk_quantize_any(int from_type, int to_type, int64_t i3 = row/(ne1*ne2); int64_t i2 = (row - i3*ne1*ne2)/ne1; int64_t i1 = row - i3*ne1*ne2 - i2*ne1; - const char * cx = (const char *)x + i1*nb1 + i2*nb2 + i3*nb3; + auto cx = (const char *)x + i1*nb1 + i2*nb2 + i3*nb3; + auto cy = (char *)y + (i3*ne1*ne2 + i2*ne1 + i1)*row_size_y; // TODO: special case common types such as f16, q8_0 // (although the performance gains may be too small to justify the added complexity) - to_float((const void *)cx, (float *)work_buffer, ne0); - auto cy = (char *)y + (i3*ne1*ne2 + i2*ne1 + i1)*row_size_y; - from_float((const float *)work_buffer, (void *)cy, ne0); + if (type_x != GGML_TYPE_F32) { + to_float((const void *)cx, (float *)work_buffer, ne0); + from_float((const float *)work_buffer, (void *)cy, ne0); + } else { + from_float((const float *)cx, (void *)cy, ne0); + } } } diff --git a/include/llama.h b/include/llama.h index 42539c7082..448fd464c3 100644 --- a/include/llama.h +++ b/include/llama.h @@ -394,6 +394,8 @@ extern "C" { int32_t n_v_first; int32_t n_v_last; + enum ggml_type extra_output_type; + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() const float * tensor_split; @@ -520,6 +522,7 @@ extern "C" { enum ggml_type ffn_down_type; // feedforward network down type enum ggml_type ffn_up_type; // feedforward network up type enum ggml_type ffn_gate_inp_type; // routed experts probabilities typy (relevant for MoE models only) + enum ggml_type extra_output_type; // routed experts probabilities typy (relevant for MoE models only) bool allow_requantize; // allow quantizing non-f32/f16 tensors bool quantize_output_tensor; // quantize output.weight bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored diff --git a/src/graphs/build_qwen35.cpp b/src/graphs/build_qwen35.cpp index 93c18793d7..2fd0eecd02 100644 --- a/src/graphs/build_qwen35.cpp +++ b/src/graphs/build_qwen35.cpp @@ -238,7 +238,7 @@ struct ggml_tensor * llm_build_context::build_qwen35moe_mtp( cb(cur, "result_norm", -1); - cur = build_output(lctx, ctx0, cur, model.output, mtp_layer.nextn.shared_head_norm, cb); + cur = build_output(lctx, ctx0, cur, model.output_mtp, mtp_layer.nextn.shared_head_norm, cb); cb(cur, "result_output", -1); return cur; @@ -317,7 +317,7 @@ struct ggml_tensor * llm_build_context::build_qwen35_mtp( cb(cur, "result_norm", -1); //cur = build_output(lctx, ctx0, cur, model.output, nullptr, cb); - cur = build_output(lctx, ctx0, cur, model.output, mtp_layer.nextn.shared_head_norm, cb); + cur = build_output(lctx, ctx0, cur, model.output_mtp, mtp_layer.nextn.shared_head_norm, cb); cb(cur, "result_output", -1); return cur; diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index b1dd9b5020..033a08dc71 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -1520,6 +1520,13 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { if (model.output == NULL) { model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } + if (model.mtp) { + model.output_mtp = create_tensor(ctx_output, "output_extra.weight", {n_embd, n_vocab}, + llama_model_loader::TENSOR_NOT_REQUIRED); + if (!model.output_mtp) { + model.output_mtp = model.output; + } + } } const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; @@ -1614,9 +1621,18 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { // output { model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); + model.output = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, + llama_model_loader::TENSOR_NOT_REQUIRED); if (model.output == NULL) { - model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); + model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, + llama_model_loader::TENSOR_DUPLICATED); + } + if (model.mtp) { + model.output_mtp = create_tensor(ctx_output, "output_extra.weight", {n_embd, n_vocab}, + llama_model_loader::TENSOR_NOT_REQUIRED); + if (!model.output_mtp) { + model.output_mtp = model.output; + } } } diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 175c20722a..f8c09c0e04 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -70,6 +70,10 @@ struct llama_model_loader { offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx); if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > file->size()) { + auto data_offset = gguf_get_data_offset(gguf_ctx); + auto tensor_offset = gguf_get_tensor_offset(gguf_ctx, tensor_idx); + fprintf(stderr, "Error while loading tensor %s: offs = %zu (%zu, %zu), size: %zu, file size: %zu\n", name, + offs, data_offset, tensor_offset, ggml_nbytes(tensor), file->size()); throw std::runtime_error(format("tensor '%s' data is not within the file bounds, model is corrupted or incomplete", name)); } } diff --git a/src/llama-model.h b/src/llama-model.h index decdbb2b56..9691b96204 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -415,6 +415,9 @@ struct llama_model { struct ggml_tensor * output; struct ggml_tensor * output_b; struct ggml_tensor * output_norm_enc; + struct ggml_tensor * output_mtp = nullptr; + + std::unique_ptr output_mtp_ptr; llama_split_tensor split_output; llama_split_tensor split_output_norm; diff --git a/src/llama-quantize.cpp b/src/llama-quantize.cpp index 6ccc054d85..eb4051bc05 100644 --- a/src/llama-quantize.cpp +++ b/src/llama-quantize.cpp @@ -924,6 +924,72 @@ static llama_ftype repacked_ftype(llama_ftype ftype) { return ftype; } +static void do_quantize(int nthread, const ggml_tensor * tensor, ggml_type new_type, const float * f32_data, char * new_data, + const float * imatrix, std::vector & workers, size_t & new_size, int chunk_size_multiplier, + const llama_model_quantize_params * params) { + if (nthread > 1 && (tensor->ne[2] % nthread == 0 || tensor->ne[2] >= 2*nthread)) { + std::mutex mutex; + int counter = 0; + bool valid = true; + auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, tensor, imatrix, user_data = params->user_data] () { + int ne2 = tensor->ne[2]; + auto row_size = ggml_row_size(new_type, tensor->ne[0]); + auto matrix_size = row_size * tensor->ne[1]; + size_t local_size = 0; + while (true) { + std::unique_lock lock(mutex); + int i02 = counter++; + if (i02 >= ne2) { + if (local_size > 0) { + new_size += local_size; + } + break; + } + lock.unlock(); + auto this_imatrix = imatrix ? imatrix + i02 * tensor->ne[0] : nullptr; + auto this_data = (char *)new_data + i02*matrix_size; + auto this_size = ggml_quantize_chunk(new_type, f32_data + i02*tensor->ne[0]*tensor->ne[1], this_data, + 0, tensor->ne[1], tensor->ne[0], this_imatrix, user_data); + local_size += this_size; + + // validate the quantized data + if (!ggml_validate_row_data(new_type, this_data, matrix_size)) { + lock.lock(); + valid = false; + break; + } + } + }; + for (int it = 0; it < nthread; ++it) workers.emplace_back(std::thread(compute)); + for (auto & w : workers) w.join(); + workers.clear(); + if (!valid) { + throw std::runtime_error("quantized data validation failed"); + } + } else { + static const int64_t min_chunk_size = 32 * 512; + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + const int64_t chunk_size = (n_per_row >= min_chunk_size + ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row)) * chunk_size_multiplier; + + const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1]; + const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size; + const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1; + + // quantize each expert separately since they have different importance matrices + new_size = 0; + for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { + const float * f32_data_03 = f32_data + i03 * nelements_matrix; + void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows; + const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; + + new_size += llama_tensor_quantize_internal(new_type, f32_data_03, new_data_03, chunk_size, + nrows, n_per_row, imatrix_03, params->user_data, workers, nthread_use); + } + } +} + static void llama_model_quantize_internal(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) { ggml_type default_type; llama_ftype ftype = params->ftype; @@ -1210,6 +1276,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s ctx_outs[0] = ctx_out; // populate the original tensors so we get an initial meta data + int last_split = -1; for (int i = 0; i < ml.n_tensors; ++i) { auto weight = ml.get_weight(i); uint16_t i_split = params->keep_split ? weight->idx : 0; @@ -1218,6 +1285,37 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s ctx_outs[i_split] = gguf_init_empty(); } gguf_add_tensor(ctx_outs[i_split], tensor); + if (i_split > last_split) { + last_split = i_split; + } + } + + ggml_tensor extra; + std::string output_tensor_name; + if (params->extra_output_type != GGML_TYPE_COUNT) { + auto meta = ml.get_tensor_meta("output.weight"); + if (!meta) { + meta = ml.get_tensor_meta("token_embd.weight"); + } + if (!meta) { + LLAMA_LOG_WARN("Extra output tensor requested, but 'output.weight' or 'token_embd.weight' not found\n"); + } else { + output_tensor_name = meta->name; + LLAMA_LOG_INFO("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Will duplicate %s as %s\n", output_tensor_name.c_str(), + ggml_type_name(params->extra_output_type)); + GGML_ASSERT(last_split >= 0); + auto weights = ml.get_weight(meta->name); + extra = *weights->tensor; + auto new_type = params->extra_output_type; + extra.type = new_type; + auto tt = ggml_internal_get_type_traits(extra.type); + extra.nb[0] = tt.type_size; + extra.nb[1] = ggml_row_size(extra.type, extra.ne[0]); + extra.nb[2] = extra.nb[3] = extra.nb[1]*extra.ne[1]; + extra.data = nullptr; + strcpy(extra.name, "output_extra.weight"); + gguf_add_tensor(ctx_outs[last_split], &extra); + } } // Set split info if needed @@ -1290,7 +1388,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s continue; } - const std::string name = ggml_get_name(tensor); + std::string name = ggml_get_name(tensor); if (!ml.use_mmap) { if (read_data.size() < ggml_nbytes(tensor)) { @@ -1450,7 +1548,10 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // If we've decided to quantize to the same type the tensor is already // in then there's nothing to do. - quantize = tensor->type != new_type; + //quantize &= tensor->type != new_type + if (name != output_tensor_name) { + quantize &= tensor->type != new_type; + } } if (!quantize) { @@ -1566,72 +1667,71 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s f32_data = (float *) f32_conv_buf.data(); } - if (work.size() < (size_t)nelements * 4) { - work.resize(nelements * 4); // upper bound on size + auto expected_size = ggml_row_size(new_type, tensor->ne[0])*tensor->ne[1]*tensor->ne[2]*tensor->ne[3]; + + if (work.size() < expected_size) { //(size_t)nelements * 4) { + //work.resize(nelements * 4); // upper bound on size + work.resize(expected_size); // upper bound on size } new_data = work.data(); - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; - - if (nthread > 1 && (tensor->ne[2] % nthread == 0 || tensor->ne[2] >= 2*nthread)) { - std::mutex mutex; - int counter = 0; - bool valid = true; - auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, tensor, imatrix, user_data = params->user_data] () { - int ne2 = tensor->ne[2]; - auto row_size = ggml_row_size(new_type, tensor->ne[0]); - auto matrix_size = row_size * tensor->ne[1]; - size_t local_size = 0; - while (true) { - std::unique_lock lock(mutex); - int i02 = counter++; - if (i02 >= ne2) { - if (local_size > 0) { - new_size += local_size; - } - break; - } - lock.unlock(); - auto this_imatrix = imatrix ? imatrix + i02 * tensor->ne[0] : nullptr; - auto this_data = (char *)new_data + i02*matrix_size; - auto this_size = ggml_quantize_chunk(new_type, f32_data + i02*tensor->ne[0]*tensor->ne[1], this_data, 0, tensor->ne[1], tensor->ne[0], - this_imatrix, user_data); - local_size += this_size; - - // validate the quantized data - if (!ggml_validate_row_data(new_type, this_data, matrix_size)) { - lock.lock(); - valid = false; - break; - } - } - }; - for (int it = 0; it < nthread; ++it) workers.emplace_back(std::thread(compute)); - for (auto & w : workers) w.join(); - workers.clear(); - if (!valid) { - throw std::runtime_error("quantized data validation failed"); + if (params->extra_output_type != GGML_TYPE_COUNT && name == output_tensor_name) { + auto cur_size = ggml_nbytes(tensor); + if (new_type != tensor->type) { + do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, + new_size, chunk_size_multiplier, params); + gguf_set_tensor_type(ctx_outs[cur_split], name.c_str(), new_type); + gguf_set_tensor_data(ctx_outs[cur_split], name.c_str(), new_data, new_size); + fout.write((const char *) new_data, new_size); + total_size_new += new_size; + LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", cur_size/1024.0/1024.0, new_size/1024.0/1024.0); + zeros(fout, GGML_PAD(new_size, align) - new_size); + } else { + gguf_set_tensor_type(ctx_outs[cur_split], name.c_str(), tensor->type); + gguf_set_tensor_data(ctx_outs[cur_split], name.c_str(), tensor->data, cur_size); + fout.write((const char *) tensor->data, cur_size); + total_size_new += cur_size; + LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", cur_size/1024.0/1024.0, cur_size/1024.0/1024.0); + } + + LLAMA_LOG_INFO("[%4d/%4d] %36s - [%s], type = %6s, ", + ++idx, ml.n_tensors, + ggml_get_name(tensor), + llama_format_tensor_shape(tensor).c_str(), + ggml_type_name(tensor->type)); + + new_type = params->extra_output_type; + chunk_size_multiplier = 1; + auto [working_type, num_rows] = interleaved_properties(new_type); + if (tensor->ne[1] % num_rows != 0) { + new_type = working_type; + } else { + chunk_size_multiplier = num_rows; } + LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type)); + fflush(stdout); + + do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, + new_size, 1, params); + + name = "output_extra.weight"; + //auto extra = *tensor; + //extra.type = new_type; + //auto tt = ggml_internal_get_type_traits(extra.type); + //extra.nb[0] = tt.type_size; + //extra.nb[1] = ggml_row_size(extra.type, extra.ne[0]); + //extra.nb[2] = extra.nb[3] = extra.nb[1]*extra.ne[1]; + //extra.data = new_data; + //strcpy(extra.name, "output_extra.weight"); + //GGML_ASSERT(ggml_nbytes(&extra) == new_size); + //gguf_add_tensor(ctx_outs[cur_split], &extra); + + //name = extra.name; } else { - static const int64_t min_chunk_size = 32 * 512; - const int64_t chunk_size = (n_per_row >= min_chunk_size ? n_per_row : n_per_row * ((min_chunk_size + n_per_row - 1)/n_per_row)) * - chunk_size_multiplier; - - const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1]; - const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size; - const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1; - - // quantize each expert separately since they have different importance matrices - new_size = 0; - for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { - const float * f32_data_03 = f32_data + i03 * nelements_matrix; - void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows; - const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; - - new_size += llama_tensor_quantize_internal(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, params->user_data, workers, nthread_use); - } + do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, + new_size, chunk_size_multiplier, params); } + } LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); } diff --git a/src/llama.cpp b/src/llama.cpp index 635a0fd19c..f399be71cb 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -2132,6 +2132,83 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) { } +static void llm_requantize_output_tensor(llama_model & model, ggml_type new_type) { + if (new_type == GGML_TYPE_COUNT || !model.output) return; + if (model.output_mtp && model.output_mtp != model.output) { + LLAMA_LOG_WARN("%s: MTP output tensor is already present => not requantizing\n", __func__); + return; + } + if (model.output->type == new_type) { + LLAMA_LOG_WARN("%s: output tensor is already of type %s => not requantizing\n", __func__, ggml_type_name(new_type)); + } + auto nbytes_orig = ggml_nbytes(model.output); + auto row_size = ggml_row_size(new_type, model.output->ne[0]); + auto nbytes_new = row_size*ggml_nrows(model.output); + if (nbytes_new >= nbytes_orig) { + LLAMA_LOG_WARN("%s: if requantized to %s the output tensor size would be %zu, which is >= the current size %zu => not requantizing\n", __func__, ggml_type_name(new_type), nbytes_new, nbytes_orig); + return; + } + + LLAMA_LOG_INFO("====== Creating extra output tensor of type %s for MTP usage. Additional memory required is %.2f MiB\n", + ggml_type_name(new_type), nbytes_new/1024./1024.); + + bool is_host = ggml_backend_buffer_is_host(model.output->buffer); + + auto tensor_data = model.output->data; + std::vector tensor_data_buf; + if (!is_host) { + tensor_data_buf.resize(nbytes_orig); + ggml_backend_tensor_get(model.output, tensor_data_buf.data(), 0, nbytes_orig); + tensor_data = tensor_data_buf.data(); + } + + auto tt_new = ggml_internal_get_type_traits(new_type); + auto new_output = std::make_unique(*model.output); + new_output->type = new_type; + new_output->nb[0] = tt_new.type_size; + new_output->nb[1] = row_size; + new_output->nb[2] = new_output->nb[1] * new_output->ne[1]; + new_output->nb[3] = new_output->nb[2] * new_output->ne[2]; + GGML_ASSERT(ggml_nbytes(new_output.get()) == nbytes_new); + new_output->buffer = ggml_backend_buft_alloc_buffer(ggml_backend_buffer_get_type(model.output->buffer), nbytes_new); + new_output->data = ggml_backend_buffer_get_base(new_output->buffer); + new_output->op = GGML_OP_NONE; + for (int j = 0; j < GGML_MAX_SRC; ++j) new_output->src[j] = nullptr; + ggml_set_name(new_output.get(), "output_extra.weight"); + ggml_backend_buffer_set_usage(new_output->buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector new_data_buf; + char * new_data = (char *)new_output->data; + if (!is_host) { + new_data_buf.resize(nbytes_new); + new_data = new_data_buf.data(); + } + + int nthread = std::max(1, std::thread::hardware_concurrency()/2); + + auto compute = [t = model.output, tensor_data, new_data, nthread, new_type] (int ith) { + std::vector work(t->ne[0]); + auto tt_orig = ggml_internal_get_type_traits(t->type); + auto tt_new = ggml_internal_get_type_traits(new_type); + iqk_quantize_any(int(t->type), int(new_type), + t->ne[0], t->ne[1], t->ne[2], t->ne[3], + t->nb[0], t->nb[1], t->nb[2], t->nb[3], + tensor_data, new_data, work.data(), tt_orig.to_float, tt_new.from_float, ith, nthread); + }; + std::vector workers(nthread-1); + for (int it = 0; it < nthread-1; ++it) workers[it] = std::thread(compute, it); + compute(nthread-1); + for (auto & w : workers) w.join(); + + if (!is_host) { + ggml_backend_tensor_set(new_output.get(), new_data, 0, nbytes_new); + } + + model.output_mtp_ptr = std::move(new_output); + model.output_mtp = model.output_mtp_ptr.get(); + +} + static void llm_prepare_mla(llama_model & model, int mla) { if (model.arch != LLM_ARCH_DEEPSEEK2 && model.arch != LLM_ARCH_GLM_DSA && model.arch != LLM_ARCH_MISTRAL4) return; const auto& hparams = model.hparams; @@ -2768,6 +2845,7 @@ static bool llm_load_tensors( const float * tensor_split, ggml_type cache_type_k, ggml_type cache_type_v, + ggml_type extra_output_type, uint32_t max_ctx_size, int n_seq_max, int n_ubatch, @@ -3364,6 +3442,9 @@ static bool llm_load_tensors( if (model.arch == LLM_ARCH_GEMMA4) { llm_scale_gate_inp_s(model, use_mmap_buffer); } + if ((model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) && extra_output_type != GGML_TYPE_COUNT) { + llm_requantize_output_tensor(model, extra_output_type); + } if (use_mmap_buffer) { for (auto & mapping : ml.mappings) { @@ -3521,7 +3602,8 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam if (!llm_load_tensors( ml, model, params.n_gpu_layers, params.mla, params.split_mode, params.main_gpu, params.max_gpu, params.tensor_split, - params.type_k, params.type_v, params.max_ctx_size, params.n_seq_max, params.n_ubatch, params.amb, params.fit_margin, + params.type_k, params.type_v, params.extra_output_type, + params.max_ctx_size, params.n_seq_max, params.n_ubatch, params.amb, params.fit_margin, params.worst_graph_tokens, params.flash_attn, params.use_mlock, params.validate_quants, params.mtp, params.fit, params.dry_run, params.progress_callback, params.progress_callback_user_data @@ -5617,6 +5699,7 @@ struct llama_model_params llama_model_default_params() { /*.n_last_k =*/ -1, /*.n_first_v =*/ -1, /*.n_last_v =*/ -1, + /*.extra_output_type =*/ GGML_TYPE_COUNT, /*.tensor_split =*/ nullptr, /*.rpc_servers =*/ nullptr, /*.progress_callback =*/ nullptr, @@ -5727,6 +5810,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { /*.ffn_down_type =*/ GGML_TYPE_COUNT, /*.ffn_up_type =*/ GGML_TYPE_COUNT, /*.ffn_gat_inp_type =*/ GGML_TYPE_COUNT, + /*.extra_output_type =*/ GGML_TYPE_COUNT, /*.allow_requantize =*/ false, /*.quantize_output_tensor =*/ true, /*.only_copy =*/ false, From c7668b75bbac502b991c8dfb0bfc74478ad82d99 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sat, 16 May 2026 05:48:14 +0000 Subject: [PATCH 2/3] Remove quantize extra output option --- examples/quantize/quantize.cpp | 9 +--- include/llama.h | 1 - src/llama-quantize.cpp | 89 +--------------------------------- src/llama.cpp | 1 - 4 files changed, 3 insertions(+), 97 deletions(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 3734a67d6e..83f797220b 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -151,7 +151,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); @@ -163,7 +163,6 @@ static void usage(const char * executable) { printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor.\n"); printf(" --token-embedding-type ggml_type: use this ggml_type for the token_embd.weight tensor.\n\n"); - printf(" --extra-output-tensor ggml_type: requantize and add output tensor of that type.\n"); printf(" --ffn-gate-inp-type ggml_type: use this ggml_type for the ffn_gate_inp tensors.\n\n"); printf(" --custom-q regex1=type1,regex2=type2...: use this to specify custom quantization type rules.\n\n"); printf(" --repack Repack all tensors to the corresponding _r4/8 variant if available.\n\n"); @@ -384,12 +383,6 @@ int main(int argc, char ** argv) { } else { usage(argv[0]); } - } else if (strcmp(argv[arg_idx], "--extra-output-tensor") == 0) { - if (arg_idx < argc-1) { - params.extra_output_type = parse_ggml_type(argv[++arg_idx]); - } else { - usage(argv[0]); - } } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) { if (arg_idx < argc-1) { params.token_embedding_type = parse_ggml_type(argv[++arg_idx]); diff --git a/include/llama.h b/include/llama.h index 448fd464c3..b847b88f8f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -522,7 +522,6 @@ extern "C" { enum ggml_type ffn_down_type; // feedforward network down type enum ggml_type ffn_up_type; // feedforward network up type enum ggml_type ffn_gate_inp_type; // routed experts probabilities typy (relevant for MoE models only) - enum ggml_type extra_output_type; // routed experts probabilities typy (relevant for MoE models only) bool allow_requantize; // allow quantizing non-f32/f16 tensors bool quantize_output_tensor; // quantize output.weight bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored diff --git a/src/llama-quantize.cpp b/src/llama-quantize.cpp index eb4051bc05..8ee24ae198 100644 --- a/src/llama-quantize.cpp +++ b/src/llama-quantize.cpp @@ -1290,34 +1290,6 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } } - ggml_tensor extra; - std::string output_tensor_name; - if (params->extra_output_type != GGML_TYPE_COUNT) { - auto meta = ml.get_tensor_meta("output.weight"); - if (!meta) { - meta = ml.get_tensor_meta("token_embd.weight"); - } - if (!meta) { - LLAMA_LOG_WARN("Extra output tensor requested, but 'output.weight' or 'token_embd.weight' not found\n"); - } else { - output_tensor_name = meta->name; - LLAMA_LOG_INFO("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Will duplicate %s as %s\n", output_tensor_name.c_str(), - ggml_type_name(params->extra_output_type)); - GGML_ASSERT(last_split >= 0); - auto weights = ml.get_weight(meta->name); - extra = *weights->tensor; - auto new_type = params->extra_output_type; - extra.type = new_type; - auto tt = ggml_internal_get_type_traits(extra.type); - extra.nb[0] = tt.type_size; - extra.nb[1] = ggml_row_size(extra.type, extra.ne[0]); - extra.nb[2] = extra.nb[3] = extra.nb[1]*extra.ne[1]; - extra.data = nullptr; - strcpy(extra.name, "output_extra.weight"); - gguf_add_tensor(ctx_outs[last_split], &extra); - } - } - // Set split info if needed if (n_split > 1) { for (size_t i = 0; i < ctx_outs.size(); ++i) { @@ -1548,10 +1520,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s // If we've decided to quantize to the same type the tensor is already // in then there's nothing to do. - //quantize &= tensor->type != new_type - if (name != output_tensor_name) { - quantize &= tensor->type != new_type; - } + quantize &= tensor->type != new_type; } if (!quantize) { @@ -1675,62 +1644,8 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } new_data = work.data(); - if (params->extra_output_type != GGML_TYPE_COUNT && name == output_tensor_name) { - auto cur_size = ggml_nbytes(tensor); - if (new_type != tensor->type) { - do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, - new_size, chunk_size_multiplier, params); - gguf_set_tensor_type(ctx_outs[cur_split], name.c_str(), new_type); - gguf_set_tensor_data(ctx_outs[cur_split], name.c_str(), new_data, new_size); - fout.write((const char *) new_data, new_size); - total_size_new += new_size; - LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", cur_size/1024.0/1024.0, new_size/1024.0/1024.0); - zeros(fout, GGML_PAD(new_size, align) - new_size); - } else { - gguf_set_tensor_type(ctx_outs[cur_split], name.c_str(), tensor->type); - gguf_set_tensor_data(ctx_outs[cur_split], name.c_str(), tensor->data, cur_size); - fout.write((const char *) tensor->data, cur_size); - total_size_new += cur_size; - LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", cur_size/1024.0/1024.0, cur_size/1024.0/1024.0); - } - - LLAMA_LOG_INFO("[%4d/%4d] %36s - [%s], type = %6s, ", - ++idx, ml.n_tensors, - ggml_get_name(tensor), - llama_format_tensor_shape(tensor).c_str(), - ggml_type_name(tensor->type)); - - new_type = params->extra_output_type; - chunk_size_multiplier = 1; - auto [working_type, num_rows] = interleaved_properties(new_type); - if (tensor->ne[1] % num_rows != 0) { - new_type = working_type; - } else { - chunk_size_multiplier = num_rows; - } - LLAMA_LOG_INFO("converting to %s .. ", ggml_type_name(new_type)); - fflush(stdout); - - do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, - new_size, 1, params); - - name = "output_extra.weight"; - //auto extra = *tensor; - //extra.type = new_type; - //auto tt = ggml_internal_get_type_traits(extra.type); - //extra.nb[0] = tt.type_size; - //extra.nb[1] = ggml_row_size(extra.type, extra.ne[0]); - //extra.nb[2] = extra.nb[3] = extra.nb[1]*extra.ne[1]; - //extra.data = new_data; - //strcpy(extra.name, "output_extra.weight"); - //GGML_ASSERT(ggml_nbytes(&extra) == new_size); - //gguf_add_tensor(ctx_outs[cur_split], &extra); - - //name = extra.name; - } else { - do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, + do_quantize(nthread, tensor, new_type, f32_data, (char *)new_data, imatrix, workers, new_size, chunk_size_multiplier, params); - } } LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", ggml_nbytes(tensor)/1024.0/1024.0, new_size/1024.0/1024.0); diff --git a/src/llama.cpp b/src/llama.cpp index f399be71cb..21afa0ec0f 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5810,7 +5810,6 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { /*.ffn_down_type =*/ GGML_TYPE_COUNT, /*.ffn_up_type =*/ GGML_TYPE_COUNT, /*.ffn_gat_inp_type =*/ GGML_TYPE_COUNT, - /*.extra_output_type =*/ GGML_TYPE_COUNT, /*.allow_requantize =*/ false, /*.quantize_output_tensor =*/ true, /*.only_copy =*/ false, From b5da743b3c6cfbb48c39acf7a61753f761a51c18 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sat, 16 May 2026 06:38:18 +0000 Subject: [PATCH 3/3] Handle interleaved types --- ggml/src/iqk/iqk_quantize.cpp | 96 +++++++++++++++++++++++++++++++---- src/llama.cpp | 12 ++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/ggml/src/iqk/iqk_quantize.cpp b/ggml/src/iqk/iqk_quantize.cpp index fd12f6b594..40df1ea6e2 100644 --- a/ggml/src/iqk/iqk_quantize.cpp +++ b/ggml/src/iqk/iqk_quantize.cpp @@ -240,6 +240,81 @@ void IQ1BNQuantizer::quantize_one_row_2bn(const float * src, block_iq2_bn * y, i } } +static inline int num_rows([[maybe_unused]] ggml_type type) { +#ifdef HAVE_FANCY_SIMD + switch (type) { + case GGML_TYPE_Q2_K_R4: + case GGML_TYPE_Q3_K_R4: + case GGML_TYPE_Q6_K_R4: + case GGML_TYPE_IQ2_K_R4: + case GGML_TYPE_IQ3_K_R4: + case GGML_TYPE_IQ4_K_R4: + case GGML_TYPE_IQ5_K_R4: + case GGML_TYPE_IQ4_KS_R4: + case GGML_TYPE_IQ5_KS_R4: + case GGML_TYPE_IQ2_XXS_R4: + case GGML_TYPE_IQ2_XS_R4: + case GGML_TYPE_IQ2_S_R4: + case GGML_TYPE_IQ3_XXS_R4: + case GGML_TYPE_IQ1_S_R4: + case GGML_TYPE_IQ1_M_R4: + case GGML_TYPE_IQ3_S_R4: return 4; + case GGML_TYPE_IQ4_NL_R4: + case GGML_TYPE_Q5_0_R4: + case GGML_TYPE_Q6_0_R4: + case GGML_TYPE_IQ2_BN_R4: + case GGML_TYPE_IQ4_XS_R8: + case GGML_TYPE_Q4_K_R4: + case GGML_TYPE_Q5_K_R4: + case GGML_TYPE_Q8_KV: + case GGML_TYPE_Q8_KV_R8: + case GGML_TYPE_Q8_K_R8: return 8; + case GGML_TYPE_Q4_0_R8: + case GGML_TYPE_Q8_0_R8: + case GGML_TYPE_Q8_1: + case GGML_TYPE_Q8_K_R16: + case GGML_TYPE_BF16_R16: return 16; + default: return 1; + } +#else + switch (type) { + case GGML_TYPE_Q2_K_R4: + case GGML_TYPE_Q3_K_R4: + case GGML_TYPE_Q4_K_R4: + case GGML_TYPE_Q5_K_R4: + case GGML_TYPE_Q6_K_R4: + case GGML_TYPE_Q5_0_R4: + case GGML_TYPE_Q6_0_R4: + case GGML_TYPE_IQ4_NL_R4: + case GGML_TYPE_IQ2_K_R4: + case GGML_TYPE_IQ3_K_R4: + case GGML_TYPE_IQ4_K_R4: + case GGML_TYPE_IQ5_K_R4: + case GGML_TYPE_IQ4_KS_R4: + case GGML_TYPE_IQ5_KS_R4: + case GGML_TYPE_IQ2_XXS_R4: + case GGML_TYPE_IQ2_XS_R4: + case GGML_TYPE_IQ2_S_R4: + case GGML_TYPE_IQ3_XXS_R4: + case GGML_TYPE_IQ3_S_R4: + case GGML_TYPE_IQ1_S_R4: + case GGML_TYPE_IQ1_M_R4: + case GGML_TYPE_IQ2_BN_R4: return 4; + case GGML_TYPE_IQ4_XS_R8: + case GGML_TYPE_Q4_0_R8: + case GGML_TYPE_Q8_0_R8: + case GGML_TYPE_Q8_KV: + case GGML_TYPE_Q8_KV_R8: + case GGML_TYPE_Q8_1: + case GGML_TYPE_Q8_K_R8: return 8; + case GGML_TYPE_Q8_K_R16: + case GGML_TYPE_BF16_R16: return 16; + default: return 1; + } +#endif +} + + } void iqk_quantize_any(int from_type, int to_type, @@ -251,24 +326,27 @@ void iqk_quantize_any(int from_type, int to_type, GGML_ASSERT(ggml_type_size(type_x) == nb0); auto type_y = ggml_type(to_type); auto row_size_y = ggml_row_size(type_y, ne0); - int64_t nrows = ne1*ne2*ne3; + auto n_interleaved = num_rows(type_y); + GGML_ASSERT(ne1 % n_interleaved == 0); + int64_t ne1i = ne1/n_interleaved; + int64_t nrows = ne1i*ne2*ne3; int64_t nrows_per_thread = (nrows + nth - 1)/nth; int64_t first_row = nrows_per_thread*ith; if (first_row >= nrows) return; int64_t last_row = std::min(first_row + nrows_per_thread, nrows); for (int64_t row = first_row; row < last_row; ++row) { - int64_t i3 = row/(ne1*ne2); - int64_t i2 = (row - i3*ne1*ne2)/ne1; - int64_t i1 = row - i3*ne1*ne2 - i2*ne1; - auto cx = (const char *)x + i1*nb1 + i2*nb2 + i3*nb3; - auto cy = (char *)y + (i3*ne1*ne2 + i2*ne1 + i1)*row_size_y; + int64_t i3 = row/(ne1i*ne2); + int64_t i2 = (row - i3*ne1i*ne2)/ne1i; + int64_t i1 = row - i3*ne1i*ne2 - i2*ne1i; + auto cx = (const char *)x + i1*n_interleaved*nb1 + i2*nb2 + i3*nb3; + auto cy = (char *)y + (i3*ne1*ne2 + i2*ne1 + i1*n_interleaved)*row_size_y; // TODO: special case common types such as f16, q8_0 // (although the performance gains may be too small to justify the added complexity) if (type_x != GGML_TYPE_F32) { - to_float((const void *)cx, (float *)work_buffer, ne0); - from_float((const float *)work_buffer, (void *)cy, ne0); + to_float((const void *)cx, (float *)work_buffer, ne0*n_interleaved); + from_float((const float *)work_buffer, (void *)cy, ne0*n_interleaved); } else { - from_float((const float *)cx, (void *)cy, ne0); + from_float((const float *)cx, (void *)cy, ne0*n_interleaved); } } } diff --git a/src/llama.cpp b/src/llama.cpp index 21afa0ec0f..ee6abd7653 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -2141,6 +2141,14 @@ static void llm_requantize_output_tensor(llama_model & model, ggml_type new_type if (model.output->type == new_type) { LLAMA_LOG_WARN("%s: output tensor is already of type %s => not requantizing\n", __func__, ggml_type_name(new_type)); } + auto [other_type, n_interleaved] = interleaved_properties(new_type); + if (model.output->ne[1] % n_interleaved != 0) { + LLAMA_LOG_WARN("%s: number of rows %ld is not a multiple of %d row interleaving for %s\n", __func__, + model.output->ne[1], n_interleaved, ggml_type_name(new_type)); + LLAMA_LOG_WARN("%s: using %s instead of %s\n", __func__, ggml_type_name(other_type), ggml_type_name(new_type)); + new_type = other_type; + n_interleaved = 1; + } auto nbytes_orig = ggml_nbytes(model.output); auto row_size = ggml_row_size(new_type, model.output->ne[0]); auto nbytes_new = row_size*ggml_nrows(model.output); @@ -2186,8 +2194,8 @@ static void llm_requantize_output_tensor(llama_model & model, ggml_type new_type int nthread = std::max(1, std::thread::hardware_concurrency()/2); - auto compute = [t = model.output, tensor_data, new_data, nthread, new_type] (int ith) { - std::vector work(t->ne[0]); + auto compute = [t = model.output, tensor_data, new_data, nthread, new_type, n_interleaved] (int ith) { + std::vector work(t->ne[0]*n_interleaved); auto tt_orig = ggml_internal_get_type_traits(t->type); auto tt_new = ggml_internal_get_type_traits(new_type); iqk_quantize_any(int(t->type), int(new_type),