diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d6807b6dd47a..184c3d59285d 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -695,7 +695,9 @@ extern "C" { void * extra; // extra things e.g. for ggml-cuda.cu - char padding[8]; + char padding[16]; + // add a struct ggml_tensor * named org_src, initialized to NULL, for keeping track of original source tensors in case of in-place operations + struct ggml_tensor * org_src; }; static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 87615921c09b..9a13f50f58a4 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1242,6 +1242,28 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra GGML_ASSERT(*cur_backend_id != -1); } + // OpenVINO currently uses ggml tensor names as graph indices. Some models (e.g. gpt-oss and + // llama4) can contain duplicate ggml tensor names, so we append node ids here to keep names + // unique. This is a temporary workaround and will be further optimized away in the future. + { + bool has_openvino_backend = false; + for (int i = 0; i < sched->n_backends; i++) { + if (strcmp(ggml_backend_name(sched->backends[i]), "OPENVINO") == 0) { + has_openvino_backend = true; + break; + } + } + + if (has_openvino_backend) { + for (int i = 0; i < graph->n_nodes; i++) { + struct ggml_tensor * node = graph->nodes[i]; + char new_name[128]; + snprintf(new_name, sizeof(new_name), "%s#%d", node->name, i); + ggml_format_name(node, "%s", new_name); + } + } + } + // pass 5: split graph, find tensors that need to be copied { int i_split = 0; @@ -1360,6 +1382,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra ggml_set_input(tensor_copy); ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor } + tensor_copy->org_src = src; tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy; SET_CAUSE(tensor_copy, "4.cpy"); } diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 48c63e4d70fa..aa9eb8f66718 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, @@ -106,14 +108,6 @@ void GgmlOvDecoder::set_input_output() { auto node_name = std::string(node->name); auto node_output_name = node_name; auto * node_output = node; - if (node->op == GGML_OP_SET_ROWS) { - // SET_ROWS updates the tensor in place. For later ov op that uses the - // the view_src of SET_ROWS, we need to make sure they get the updated tensor - // by putting the view_src name in the tensor_map in - // /src/frontends/ggml/src/translate_session.cpp - node_output_name = std::string(node->view_src->name); - node_output = node->view_src; - } current_node_info.node = node; current_node_info.node_name = node_name; @@ -787,6 +781,42 @@ std::map> GgmlOvDecoder::create_weight_no return model_weights; } +// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the +// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such +// tensors have no OV buffer context to own a cached extra, so without this they are +// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB +// F32 dequant each time. Keyed by tensor->data, which is stable for the process and +// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the +// per-tensor extra cache and never reach here. +static std::mutex g_nonov_weight_cache_mutex; +static std::unordered_map> g_nonov_weight_cache; + +std::set GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) { + // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes, + // so topology checks don't trigger weight extraction/requantization. + std::set names; + for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) { + auto * node = cgraph->nodes[node_i]; + for (int i = 0; i < GGML_MAX_SRC; i++) { + auto * src = node->src[i]; + if (src == nullptr) { + continue; + } + std::string src_name(src->name); + if (is_rope_freqs_weight(src, node)) { + src_name = "rope_freqs.weight"; + } + if (!src->view_src) { + ggml_backend_buffer * buffer = src->buffer; + if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) { + names.insert(src_name); + } + } + } + } + return names; +} + std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) { const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer); @@ -826,6 +856,21 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor return weight_node; } + // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer + // context to cache an extra in, so memoize them here keyed by their (stable) data + // pointer to avoid re-extracting on every recompile. Opt-in via + // GGML_OPENVINO_REDUCE_COMPILE_MEM. Skip for `naive` (test/naive path) since use_bias + // changes the produced node. + const bool cacheable_nonov = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_ov_buffer && + !naive && tensor->data != nullptr; + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + auto it = g_nonov_weight_cache.find(tensor->data); + if (it != g_nonov_weight_cache.end()) { + return it->second; + } + } + // There are three cases where we need to create a new weight node: // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used @@ -834,7 +879,7 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name); static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, - GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}; + GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; if (weight_types.find(tensor->type) == weight_types.end()) { throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " + ggml_type_name(tensor->type)); @@ -863,6 +908,12 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor ov_weight.weight_node->set_friendly_name(tensor->name); if (!is_ov_buffer) { + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + // Another thread may have inserted concurrently; keep the first. + auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node); + return it->second; + } return ov_weight.weight_node; } @@ -1231,6 +1282,14 @@ std::vector GgmlOvDecoder::get_output_names(int node_idx) const { return {m_node_info_list[node_idx].node_output_name}; } +std::vector GgmlOvDecoder::get_output_aliases(int node_idx) const { + const auto * node = m_node_info_list[node_idx].node; + if (node != nullptr && node->op == GGML_OP_SET_ROWS && node->view_src != nullptr) { + return {std::string(node->view_src->name)}; + } + return {}; +} + const std::string & GgmlOvDecoder::get_op_name() const { static const std::string unknown_name = "UNKNOWN_OP_NAME"; return unknown_name; @@ -1307,10 +1366,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { if (src == nullptr) { continue; } - struct ggml_tensor * root_src = nullptr; - // if (src->org_src) { - // root_src = src->org_src; - // } + ggml_tensor * root_src = nullptr; + if (src->org_src) { + root_src = src->org_src; + } if (root_src) { if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { m_node_dynamic_dims[root_src] = 0; @@ -1388,7 +1447,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { // identifies the dynamic dim even when two dims share the same size. m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { - if (node->src[0]->op == GGML_OP_NONE) { + if (node->src[0]->op == GGML_OP_NONE && node->src[0]->org_src == nullptr) { m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; break; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index ae545f47e5fe..c27d3306f07f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include struct ModelParams { @@ -156,6 +158,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual std::vector get_output_names(int node_idx) const override; + virtual std::vector get_output_aliases(int node_idx) const override; + virtual const std::string & get_op_type() const override; virtual const std::string & get_op_type(int node_idx) const override; @@ -235,6 +239,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { static std::map> create_weight_nodes(ggml_cgraph * cgraph, bool naive = false); + // Collect just the set of weight-tensor names referenced by the graph, without + // building (or requantizing) any OV weight nodes. Used by topology checks like + // is_model_splitted that only need name membership. + static std::set collect_weight_names(ggml_cgraph * cgraph); + const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const; const ggml_tensor * get_tensor_from_name(const std::string & name) const; @@ -267,7 +276,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; + return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE && op->src[0]->org_src == nullptr; } inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index d9ad7be734d1..cb62a0022313 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -45,6 +45,9 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_RELEASE_WEIGHTS", + "GGML_OPENVINO_REDUCE_COMPILE_MEM", + "GGML_OPENVINO_MODEL_CACHE_DIR", }; for (const char * const & env_var : env_var_names) { @@ -252,14 +255,24 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // Only handle 2D weight tensors - if (tensor->ne[2] != 1 || tensor->ne[3] != 1) { + // Most quantized weights use the existing 2D extraction path. MXFP4 also + // appears as 3D expert weights for MUL_MAT_ID, so allow that type through. + if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) { return layout; } int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD + if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) { + layout.weights_per_block = 32; + layout.is_symmetric = true; + layout.weights_size = ggml_nbytes(tensor); + layout.weights_offset = 0; + layout.total_size = layout.weights_size; + return layout; + } + // Check if requantization is needed (NPU-specific) auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias); if (requant_type.has_value()) { @@ -334,6 +347,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = false; switch (tensor->type) { + case GGML_TYPE_MXFP4: + layout.is_u4 = true; + layout.is_symmetric = true; + break; + case GGML_TYPE_Q4_0: layout.is_u4 = true; layout.is_symmetric = true; @@ -369,9 +387,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements; - // Scales: F16 per block + // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; - layout.scales_size = n_blocks * sizeof(uint16_t); // F16 = 2 bytes + layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); // For symmetric quantization, no zp needed (weights stored as signed) if (layout.is_symmetric) { layout.zp_size = 0; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index c2654fbfa1b8..e62d966accc0 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -99,6 +99,14 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0); // Check if running on NPU bool ggml_openvino_is_npu(); +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only). +// register: record a host weight buffer (idempotent per data pointer). +// release: madvise(MADV_DONTNEED) all registered buffers, dropping their RSS. +// released: true once release has run (used to fail-fast on post-release recompile). +void ggml_openvino_register_weight_buffer(void * data, size_t size); +void ggml_openvino_release_weight_buffers(); +bool ggml_openvino_weight_buffers_released(); + // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 659dbd4b5acb..a137aaab2512 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -32,6 +32,7 @@ # endif # include #else +# include # include #endif @@ -135,6 +136,81 @@ struct ggml_backend_openvino_buffer_type_context { std::string name; }; +// ===================================================== +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS) +// ===================================================== +// The OpenVINO weight Constants are zero-copy views into the host buffers +// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin +// holds its own device copy after compile_model, so the host pages are dead +// weight for inference and can be dropped to reclaim RSS (~weights size). +// +// We do NOT free the buffer (ggml owns its lifetime and tensors still point +// into it); instead madvise(MADV_DONTNEED) drops the resident pages while +// keeping the mapping valid. A later recompile would re-read these Constants +// from now-zeroed memory and produce garbage, so once released we fail fast +// if the cache-miss compile branch is reached again (see utils.cpp). +namespace { +struct ov_weight_buffer_registry { + std::mutex mutex; + // (data, size) of every non-remote weight buffer, for madvise. + std::vector> buffers; + bool released = false; +}; + +ov_weight_buffer_registry & ov_weight_registry() { + static ov_weight_buffer_registry reg; + return reg; +} +} // namespace + +void ggml_openvino_register_weight_buffer(void * data, size_t size) { + if (data == nullptr || size == 0) { + return; + } + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + for (const auto & b : reg.buffers) { + if (b.first == data) { + return; // already registered + } + } + reg.buffers.emplace_back(data, size); +} + +bool ggml_openvino_weight_buffers_released() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + return reg.released; +} + +void ggml_openvino_release_weight_buffers() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + if (reg.released) { + return; + } + size_t total = 0; +#if !defined(_WIN32) + for (const auto & b : reg.buffers) { + // Align down/up to page boundaries so madvise only drops whole pages + // fully owned by this buffer. + const long page = sysconf(_SC_PAGESIZE); + uintptr_t start = reinterpret_cast(b.first); + uintptr_t end = start + b.second; + uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1); + uintptr_t aend = end & ~(uintptr_t) (page - 1); + if (aend > astart) { + if (madvise(reinterpret_cast(astart), aend - astart, MADV_DONTNEED) == 0) { + total += aend - astart; + } + } + } +#endif + reg.released = true; + GGML_LOG_INFO("%s: released %zu MB of host weight buffers (%zu buffers)\n", __func__, total / 1024 / 1024, + reg.buffers.size()); +} + // Buffer interface functions static void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context; @@ -237,8 +313,9 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr); // 2D tensor (typical weight shape) bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1); + bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4; - if (is_weight_buffer && is_full_tensor_set && is_2d) { + if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) { try { auto result = process_weight_tensor(tensor, data, tensor->data); result.weight_node->set_friendly_name(tensor->name); @@ -274,6 +351,22 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer ctx->tensor_extras[tensor] = extra; tensor->extra = extra; + // Register the host buffer so its pages can be dropped after the GPU + // plugin has its own device copy (GGML_OPENVINO_RELEASE_WEIGHTS). + if (!ctx->is_remote) { + // Weights are set once at model load. Setting a weight after a release + // means a second model is loading while the first's compiled graph is + // pinned — that graph would be wrongly reused with this model's key. + // Fail loud rather than return silently-wrong results. + if (ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: loading a new model while GGML_OPENVINO_RELEASE_WEIGHTS pinned a previous " + "model's compiled graph. This mode supports a single model per process; unset it for " + "multi-model runs."); + } + ggml_openvino_register_weight_buffer(ctx->data, ctx->size); + } + } catch (const std::exception & e) { GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what()); memcpy((char *) tensor->data + offset, data, size); @@ -458,8 +551,9 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff const ggml_tensor * tensor) { GGML_UNUSED(buft); - // For quantized 2D tensors (weights), we need extra space for extracted data - if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) { + // For quantized weight tensors, we need extra space for extracted data. + if (ggml_is_quantized(tensor->type) && + ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) { ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", @@ -618,7 +712,13 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) { if (ctx->runtime_context) { auto r_ctx = std::static_pointer_cast(ctx->runtime_context); if (--r_ctx->backend_count == 0) { - r_ctx->clear_caches(); + // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the + // dropped pages can never be repopulated, so a recompile is impossible. Keep + // the compiled-model cache alive across backend teardown so the next context + // reuses it instead of recompiling against zeroed weights. + if (!ggml_openvino_weight_buffers_released()) { + r_ctx->clear_caches(); + } } } @@ -901,17 +1001,10 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { return true; } - // Keep the MoE routing weights gather on CPU for GPU runs. Splitting - // only at the later SUM/CLAMP/DIV nodes still leaves this routing path - // numerically unstable for arctic-style MoE graphs. - if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { - return true; - } break; } case GGML_OP_RESHAPE: { - if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 || - strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { + if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { return true; } break; @@ -938,69 +1031,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_DIV: { - bool requires_broadcast = false; - for (int i = 0; i < 4; i++) { - if (op->src[0]->ne[i] == op->src[1]->ne[i]) { - continue; - } - - if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) { - return true; - } - - requires_broadcast = true; - } - // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path // and produce infs for per-channel scale vectors. Keep those DIVs on CPU // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt) - if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") { - return true; - } - - // qwen3next MoE weight normalization is numerically sensitive on the GPU - // path. Keep the normalization divide on CPU to match the reference. - if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) { - return true; - } - break; - } - case GGML_OP_SOFT_MAX: { - if (op->src[2] != nullptr) { - // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n"); - return true; - } - - if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) { - return true; - } - - // GPU execution of the MoE routing weights softmax is numerically unstable - // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax - // on CPU so the scheduler splits at the same boundary that restores parity. - if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr && - strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { + if (op->src[1]->ne[0] == 1 && op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 384) { return true; } break; } case GGML_OP_SUM_ROWS: { - if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) { - return true; - } - // if the input is PERMUTE skip if (op->src[0]->op == GGML_OP_PERMUTE) { return true; } break; } - case GGML_OP_CLAMP: { - if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) { - return true; - } - break; - } case GGML_OP_FLASH_ATTN_EXT: { float scale = 1.0f; float max_bias = 0.0f; @@ -1060,12 +1105,6 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT: { - if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX && - op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr && - op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr && - op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { - return true; - } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return true; } @@ -1075,12 +1114,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_MUL_MAT_ID: { - if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 || - strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) { - return true; - } - - if (mul_mat_id_requires_large_tmp(op)) { + if (mul_mat_id_requires_large_tmp(op) && + !(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) { return true; } break; @@ -1158,8 +1193,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { return true; } case GGML_OP_VIEW: { - // Skip TOPK_MOE fused tests until it is fully supported - // the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe + // Skip TOPK_MOE fused tests until it is fully supported. + // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe. if (strcmp(op->name, "selected_experts") == 0) { return true; } @@ -1176,7 +1211,8 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con static std::unordered_set supported_types{ GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, - GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; + GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, + GGML_TYPE_MXFP4}; // derive supported op sets from the op_table map, keys in // the map use the full macro name (e.g. "GGML_OP_ADD"), while @@ -1274,7 +1310,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type)); return false; } - if (ggml_is_quantized(src->type) && src->ne[2] != 1) { + const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 && + src->type == GGML_TYPE_MXFP4; + if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) { // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); return false; } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 275b95428273..fa48197d94c5 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -2,6 +2,7 @@ #include "ggml-common.h" #include "ggml-impl.h" +#include "ggml-openvino-extra.h" #include "ggml.h" #include @@ -18,7 +19,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -44,6 +47,38 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) { } } +static constexpr size_t MXFP4_BLOCK_SIZE = 32; +static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; +static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; + +static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { + for (int j = 0; j < static_cast(MXFP4_BLOCK_QS_SIZE); j += 2) { + const uint8_t v0 = data[j] & 0x0F; + const uint8_t v1 = (data[j + 1] & 0x0F) << 4; + const uint8_t v16 = data[j] >> 4; + const uint8_t v17 = data[j + 1] & 0xF0; + dst[j / 2] = v0 | v1; + dst[MXFP4_BLOCK_SIZE / 4 + j / 2] = v16 | v17; + } +} + +void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) { + GGML_ASSERT(tensor->type == GGML_TYPE_MXFP4); + GGML_ASSERT(weights_arr.get_element_type() == ov::element::f4e2m1); + GGML_ASSERT(scales_arr.get_element_type() == ov::element::f8e8m0); + + const auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + const size_t n_blocks = scales_arr.get_size(); + + ov::parallel_for(n_blocks, [&](size_t i) { + const uint8_t * block = data + i * MXFP4_BLOCK_BYTES; + pack_32_mxfp4_for_openvino(block + sizeof(uint8_t), weights + i * MXFP4_BLOCK_QS_SIZE); + scales[i] = ov::float8_e8m0::from_bits(block[0]); + }); +} + // Extracts (weight, scales, zp) from Q4_0 tensors. // Data layout is: |16 bit scale|32 x 4bit weights|. // When zp_arr is empty (symmetric), weights are stored as signed i4 (value - 8). @@ -617,6 +652,42 @@ ov::Output make_int4_weights(ov::Tensor & weight, return std::make_shared(result, ov::element::f32); } +ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales) { + const ov::Shape final_shape = weight.get_shape(); + GGML_ASSERT(!final_shape.empty()); + GGML_ASSERT(final_shape.back() % MXFP4_BLOCK_SIZE == 0); + + ov::Shape packed_shape = final_shape; + packed_shape.back() /= MXFP4_BLOCK_SIZE; + packed_shape.push_back(MXFP4_BLOCK_SIZE); + + ov::Shape scale_shape = packed_shape; + scale_shape.back() = 1; + scales.set_shape(scale_shape); + + auto weights_node = std::make_shared(ov::element::f4e2m1, packed_shape, + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + auto weights_f32 = std::make_shared(weights_node, ov::element::f32); + + auto scales_node = std::make_shared(scales); + auto scales_f32 = std::make_shared(scales_node, ov::element::f32); + ov::Output result = + std::make_shared(weights_f32, scales_f32, ov::op::AutoBroadcastType::NUMPY); + + auto final_shape_node = + std::make_shared(ov::element::i64, ov::Shape{final_shape.size()}, final_shape); + return std::make_shared(result, final_shape_node, false); +} + +ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight) { + auto weights_node = std::make_shared(ov::element::u8, weight.get_shape(), + static_cast(weight.data()), nullptr); + weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; + weights_node->get_rt_info()["__ggml_openvino_mxfp4_moe_packed"] = true; + return weights_node; +} + // Extract quantized weights from tensor and create weight subgraph std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, const void * data, @@ -628,6 +699,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, ggml_tensor temp_tensor = *tensor; temp_tensor.data = const_cast(data); + if (tensor->type == GGML_TYPE_MXFP4) { + extract_mxfp4_data(&temp_tensor, weights, scales); + auto result = make_mxfp4_weights(weights, scales).get_node_shared_ptr(); + result->set_friendly_name(tensor->name); + return result; + } + // Determine block size based on tensor type int64_t weights_per_block; bool is_u4; @@ -702,28 +780,75 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, ov::Tensor & scales, ov::Tensor & zp) { int64_t n_elements = ggml_nelements(tensor); + const int64_t ne0 = tensor->ne[0]; // elements per row + const int64_t n_rows = n_elements / ne0; + const auto * type_traits = ggml_get_type_traits(tensor->type); + const size_t src_row_bytes = ggml_row_size(tensor->type, ne0); - // First dequantize to F32 - std::vector weights_f32(n_elements); - ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements); - - // Handle F16 case - just convert and create constant - if (requant_type == ExtraQuantType::F16) { - ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); - auto result = std::make_shared(weights); - result->set_friendly_name(tensor->name); - return result; - } - - // Requantize to target quantized format bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); - if (is_u4) { - quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); - } else if (requant_type == ExtraQuantType::Q8_1_C) { - quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM): instead of + // materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize + // a chunk of complete rows into a small scratch and quantize/convert it straight into + // the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats. + // + // Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size + // divides a row (channel-wise _C uses block_size == ne0) so no target block straddles + // a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two + // weights per byte with running zp ORs that assume a single whole-array call, so it is + // never streamed. When the flag is off, behavior is identical to the original + // full-materialization path. + const bool stream_requant = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_u4 && + !(block_size > 0 && ne0 % block_size != 0); + + if (!stream_requant) { + // Full materialization (original behavior): dequantize the whole tensor to F32, + // then convert/quantize in one call. + std::vector weights_f32(n_elements); + type_traits->to_float(data, weights_f32.data(), n_elements); + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } + if (is_u4) { + quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else { + quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } } else { - quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + // Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight, + // and per-layer Q6_K/Q5_K requant — the large transient cases). + const int64_t CHUNK_ROWS = std::min(n_rows, 256); + std::vector scratch(CHUNK_ROWS * ne0); + // F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements. + auto * f16_base = static_cast(weights.data()); + for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) { + const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0); + const int64_t elems = rows * ne0; + const auto * src = static_cast(data) + r0 * src_row_bytes; + type_traits->to_float(src, scratch.data(), elems); + + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16) + ->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems); + } else { + const int64_t block_offset = (r0 * ne0) / block_size; + if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } else { + quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } + } + } + if (requant_type == ExtraQuantType::F16) { + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } } // Create the OpenVINO weight subgraph @@ -788,6 +913,27 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } + const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); + if (is_3d_mxfp4_moe) { + ov::Shape packed_shape = {static_cast(tensor->ne[3]), + static_cast(tensor->ne[2]), + static_cast(tensor->ne[1]), + static_cast(tensor->ne[0] / MXFP4_BLOCK_SIZE), + MXFP4_BLOCK_BYTES}; + const size_t tensor_bytes = ggml_nbytes(tensor); + if (output_base_ptr) { + auto * buf_base = static_cast(output_base_ptr); + memcpy(buf_base + layout.weights_offset, data, tensor_bytes); + result.weights = ov::Tensor(ov::element::u8, packed_shape, buf_base + layout.weights_offset); + } else { + result.weights = ov::Tensor(ov::element::u8, packed_shape); + memcpy(result.weights.data(), data, tensor_bytes); + } + result.weight_node = make_mxfp4_moe_packed_weights(result.weights).get_node_shared_ptr(); + result.weight_node->set_friendly_name(tensor->name); + return result; + } + if (use_bias) { OPENVINO_ASSERT(!layout.is_requant, "use_bias is only used for test-backend-ops, which should not have requantization"); @@ -812,14 +958,31 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // Quantized path (normal extraction or quantized requant) // Create weight/scale/zp tensors - shared between both paths // For symmetric quantization, use signed types (i4/i8) and no ZP tensor - ov::element::Type weight_type = layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : - (layout.is_u4 ? ov::element::u4 : ov::element::u8); + ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ? + ov::element::f4e2m1 : + (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : + (layout.is_u4 ? ov::element::u4 : ov::element::u8)); ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block}; + if (tensor->type == GGML_TYPE_MXFP4) { + if (tensor->ne[2] == 1 && tensor->ne[3] == 1) { + node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; + } else { + node_shape.clear(); + for (int i = GGML_MAX_DIMS - 1; i >= 0; --i) { + node_shape.push_back(static_cast(tensor->ne[i])); + } + } + + scale_shape = node_shape; + scale_shape.back() /= layout.weights_per_block; + } + if (output_base_ptr) { uint8_t * buf_base = static_cast(output_base_ptr); result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset); - result.scales = ov::Tensor(ov::element::f16, scale_shape, buf_base + layout.scales_offset); + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); @@ -827,7 +990,8 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // else: result.zp remains default-constructed (empty) for symmetric } else { result.weights = ov::Tensor(weight_type, node_shape); - result.scales = ov::Tensor(ov::element::f16, scale_shape); + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + result.scales = ov::Tensor(scale_type, scale_shape); if (!layout.is_symmetric) { if (use_bias) { result.zp = ov::Tensor(ov::element::f16, scale_shape); @@ -939,16 +1103,21 @@ void quantize_q8_0(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk) { + int64_t qk, + int64_t block_offset) { assert(k % qk == 0); const int nb = k / qk; - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); + // block_offset lets a caller quantize a chunk of blocks into the right place in the + // output buffers (used for streaming requant). x points at this chunk's first block; + // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no + // nibble packing), so any block boundary is safe. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path if (!is_symmetric) { - auto * zp = static_cast(zp_arr.data()); + auto * zp = static_cast(zp_arr.data()) + block_offset; for (int i = 0; i < nb; i++) { float amax = 0.0f; for (int j = 0; j < qk; j++) { @@ -990,13 +1159,15 @@ void quantize_q8_1(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk) { + int64_t qk, + int64_t block_offset) { assert(k % qk == 0); const int nb = k / qk; - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); + // See quantize_q8_0: block_offset places this chunk's output at the right block. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + auto * zp = static_cast(zp_arr.data()) + block_offset; for (int i = 0; i < nb; i++) { float min = std::numeric_limits::max(); float max = std::numeric_limits::lowest(); diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index 28b7c1213be2..fddbd9fd2b32 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -4,6 +4,7 @@ #include #include +#include #include void unpack_32_4(const uint8_t * data, uint8_t * dst); @@ -49,6 +50,8 @@ void extract_q6_k_data(const ggml_tensor * tensor, ov::Tensor & scales_arr, ov::Tensor & zp_arr); +void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr); + static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; ov::Output make_int8_weights(ov::Tensor & weight, @@ -63,6 +66,10 @@ ov::Output make_int4_weights(ov::Tensor & weight, size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, bool use_bias = false); +ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales); + +ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight); + // Extract quantized weights from tensor and create weight subgraph // If weights/scales/zp are provided (non-empty), uses them as output buffers // Otherwise allocates new ov::Tensors internally @@ -139,13 +146,15 @@ void quantize_q8_1(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk); + int64_t qk, + int64_t block_offset = 0); void quantize_q8_0(const float * x, ov::Tensor & weights_arr, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk); + int64_t qk, + int64_t block_offset = 0); namespace ov { namespace op { diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp new file mode 100644 index 000000000000..2c2e5ac14d0f --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -0,0 +1,272 @@ +#include "model-cache.h" + +#include "ggml-backend-impl.h" +#include "ggml-backend.h" +#include "ggml-impl.h" +#include "ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# include +#endif + +namespace { + +// 64-bit FNV-1a, the mixing primitive for all fingerprints here. +inline uint64_t fnv1a(uint64_t h, const void * data, size_t n) { + const uint8_t * p = static_cast(data); + for (size_t i = 0; i < n; ++i) { + h ^= p[i]; + h *= 0x100000001b3ull; + } + return h; +} + +inline uint64_t fnv1a_u64(uint64_t h, uint64_t v) { + return fnv1a(h, &v, sizeof(v)); +} + +constexpr uint64_t FNV_OFFSET = 0xcbf29ce484222325ull; + +// Bytes sampled from each end of a weight tensor for the sampled hash. The whole +// model is never hashed (that would cost seconds every run); instead we sample a +// bounded window from the head and tail of each weight's bytes. The manifest +// re-verify (same sample) guards the residual collision risk. +constexpr size_t WEIGHT_SAMPLE_BYTES = 4096; + +// Is this src a model weight, mirroring create_weight_nodes()'s selection: +// non-view tensor whose buffer is USAGE_WEIGHTS or whose type is quantized. +bool is_weight_src(const ggml_tensor * src) { + if (src == nullptr || src->view_src != nullptr || src->buffer == nullptr) { + return false; + } + return src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type); +} + +// Per-weight sampled fingerprint: identity (name/shape/type) + a bounded byte +// sample. Returns FNV offset basis if data is unavailable (kept deterministic). +uint64_t weight_fingerprint(const ggml_tensor * t) { + uint64_t h = FNV_OFFSET; + h = fnv1a(h, t->name, strlen(t->name)); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + h = fnv1a_u64(h, static_cast(t->ne[i])); + } + h = fnv1a_u64(h, static_cast(t->type)); + const size_t nbytes = ggml_nbytes(t); + h = fnv1a_u64(h, nbytes); + if (t->data != nullptr && nbytes > 0) { + const size_t head = nbytes < WEIGHT_SAMPLE_BYTES ? nbytes : WEIGHT_SAMPLE_BYTES; + h = fnv1a(h, t->data, head); + if (nbytes > WEIGHT_SAMPLE_BYTES) { + const size_t tail = nbytes < 2 * WEIGHT_SAMPLE_BYTES ? nbytes - WEIGHT_SAMPLE_BYTES : WEIGHT_SAMPLE_BYTES; + h = fnv1a(h, static_cast(t->data) + (nbytes - tail), tail); + } + } + return h; +} + +// Walk the cgraph and invoke fn(weight_tensor) for each distinct weight, in node +// order. De-duplicates by tensor pointer so a weight used by several nodes is +// fingerprinted once, deterministically. +template +void for_each_weight(const ggml_cgraph * cgraph, F && fn) { + std::vector seen; + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = node->src[s]; + if (!is_weight_src(src)) { + continue; + } + bool dup = false; + for (const auto * p : seen) { + if (p == src) { + dup = true; + break; + } + } + if (dup) { + continue; + } + seen.push_back(src); + fn(src); + } + } +} + +std::string ov_version_string() { + const ov::Version v = ov::get_openvino_version(); + return std::string(v.buildNumber ? v.buildNumber : "unknown"); +} + +std::string hex64(uint64_t v) { + char buf[17]; + snprintf(buf, sizeof(buf), "%016llx", static_cast(v)); + return std::string(buf); +} + +// Portable mkdir for a single path component. Returns true if the directory +// exists after the call (created now or already present). +bool make_dir(const std::string & path) { +#if defined(_WIN32) + int rc = _mkdir(path.c_str()); +#else + int rc = ::mkdir(path.c_str(), 0755); +#endif + if (rc == 0 || errno == EEXIST) { + return true; + } + return false; +} + +// Create `path` and any missing parents (like `mkdir -p`). Best-effort: +// returns true only if the full directory exists afterwards. +bool make_dirs(const std::string & path) { + if (path.empty()) { + return false; + } + std::string acc; + for (size_t i = 0; i < path.size(); ++i) { + const char c = path[i]; + acc.push_back(c); + const bool sep = (c == '/' +#if defined(_WIN32) + || c == '\\' +#endif + ); + // Create each intermediate component (skip a leading "/" root). + if (sep && acc.size() > 1) { + std::string component = acc.substr(0, acc.size() - 1); + if (!make_dir(component)) { + return false; + } + } + } + return make_dir(path); +} + +} // namespace + +std::string ggml_openvino_model_cache_dir() { + const char * dir = ggml_openvino_getenv_str("GGML_OPENVINO_MODEL_CACHE_DIR"); + if (!dir || strlen(dir) == 0) { + return std::string(); + } + std::string path(dir); + // Create the cache directory (and parents) on first use so callers don't + // have to pre-create it; a missing dir would otherwise silently disable the + // cache (manifest/blob writes fail with no directory to write into). + if (!make_dirs(path)) { + GGML_LOG_WARN("ggml-openvino: could not create model cache dir '%s' (errno=%d); caching disabled\n", + path.c_str(), errno); + return std::string(); + } + return path; +} + +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg) { + uint64_t h = FNV_OFFSET; + + // Topology: node count + each node's op and name (cheap, and distinguishes + // graphs that share weights but differ structurally). + h = fnv1a_u64(h, static_cast(cgraph->n_nodes)); + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + h = fnv1a_u64(h, static_cast(node->op)); + h = fnv1a(h, node->name, strlen(node->name)); + } + + // Weights: the model identity. + for_each_weight(cgraph, [&](const ggml_tensor * t) { h = fnv1a_u64(h, weight_fingerprint(t)); }); + + // Config that changes the produced blob. + h = fnv1a(h, device.data(), device.size()); + h = fnv1a_u64(h, fa ? 1u : 0u); + if (rope_params && rope_len > 0) { + h = fnv1a(h, rope_params, sizeof(int32_t) * static_cast(rope_len)); + } + h = fnv1a_u64(h, extra_cfg); + const std::string ver = ov_version_string(); + h = fnv1a(h, ver.data(), ver.size()); + + return h; +} + +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".blob"; +} + +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".manifest"; +} + +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint) { + std::ofstream f(path, std::ios::trunc); + if (!f.is_open()) { + return false; + } + f << "fingerprint " << hex64(fingerprint) << "\n"; + f << "ov_version " << ov_version_string() << "\n"; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + f << t->name << " " << t->ne[0] << " " << t->ne[1] << " " << t->ne[2] << " " << t->ne[3] << " " + << static_cast(t->type) << " " << hex64(weight_fingerprint(t)) << "\n"; + }); + return f.good(); +} + +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint) { + std::ifstream f(path); + if (!f.is_open()) { + return false; + } + std::string tag, val; + // header: fingerprint + if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) { + return false; + } + // header: ov_version + if (!(f >> tag >> val) || tag != "ov_version" || val != ov_version_string()) { + return false; + } + + // Build the expected per-weight lines from the live cgraph, then require an + // exact match (same set, same order) against the manifest. + std::vector expected; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + expected.push_back(std::string(t->name) + " " + std::to_string(t->ne[0]) + " " + std::to_string(t->ne[1]) + + " " + std::to_string(t->ne[2]) + " " + std::to_string(t->ne[3]) + " " + + std::to_string(static_cast(t->type)) + " " + hex64(weight_fingerprint(t))); + }); + + size_t idx = 0; + std::string line; + std::getline(f, line); // consume rest of ov_version line + while (std::getline(f, line)) { + if (line.empty()) { + continue; + } + if (idx >= expected.size() || line != expected[idx]) { + return false; + } + ++idx; + } + return idx == expected.size(); +} diff --git a/ggml/src/ggml-openvino/model-cache.h b/ggml/src/ggml-openvino/model-cache.h new file mode 100644 index 000000000000..ec46e827a448 --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.h @@ -0,0 +1,56 @@ +#pragma once + +// Frontend-level model cache (GGML_OPENVINO_MODEL_CACHE_DIR). +// +// The OpenVINO plugin's own ov::cache_dir caches the compiled blob keyed by the +// *OV model*, but producing that model still runs the full frontend every time: +// weight requantization (incl. the large token_embd F32 transient) and the +// ggml->OV graph conversion. This cache keys off a fingerprint computed directly +// from the ggml cgraph, so a hit skips requant + convert + compile entirely and +// instead imports a previously exported CompiledModel blob. +// +// Opt-in and independent from GGML_OPENVINO_CACHE_DIR. Default off. + +#include "ggml.h" + +#include +#include + +// Returns the model-cache directory from GGML_OPENVINO_MODEL_CACHE_DIR, or empty +// if unset/disabled. When empty, callers must not use the cache. +std::string ggml_openvino_model_cache_dir(); + +// Compute a stable 64-bit fingerprint identifying the model+config that a cgraph +// would compile to. Combines graph topology, a sampled hash of every weight +// tensor (name/shape/dtype + bounded byte sample), and the config that changes +// the produced blob (device, flash-attention, rope params, the compile-memory +// flags, stateful, and the OpenVINO version). `device` is the resolved device +// string; `fa` is the flash-attention flag; `rope_params`/`rope_len` cover the +// model's rope configuration; `extra_cfg` folds in any other blob-affecting bits. +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg); + +// Path to the compiled-blob file for a fingerprint (/.blob). +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint); + +// Path to the sidecar manifest (/.manifest) holding the per-weight +// fingerprints, used to re-verify a hit before trusting the blob. +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint); + +// Write/read the manifest. The manifest is a newline-separated list of +// "name ne0 ne1 ne2 ne3 type sample_hash" lines plus a header line with the +// fingerprint and OV version. Returns false on I/O error. +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint); + +// Verify that the cgraph's weights still match the stored manifest (guards the +// sampled-hash collision risk: a blob is only trusted if every weight's +// name/shape/type/sample-hash matches what was cached). Returns true on match. +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint); diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index 9d64fe575c4c..3b429078c343 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -75,6 +75,8 @@ class GgmlDecoder : public DecoderBase { virtual std::vector get_output_names(int node_idx) const = 0; + virtual std::vector get_output_aliases(int node_idx) const = 0; + virtual const std::string & get_op_type() const = 0; virtual const std::string & get_op_type(int node_idx) const = 0; diff --git a/ggml/src/ggml-openvino/openvino/op/fill.cpp b/ggml/src/ggml-openvino/openvino/op/fill.cpp new file mode 100644 index 000000000000..1450b70be23d --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/fill.cpp @@ -0,0 +1,47 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_fill(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "FILL requires output op params"); + + float value; + std::memcpy(&value, op_params, sizeof(float)); + + auto scalar = ov::op::v0::Constant::create(context.get_output_type(), ov::Shape{}, {value}); + + ov::Output target_shape; + const auto output_shape = context.get_output_shape(); + if (output_shape.rank().is_static() && output_shape.is_static()) { + const auto static_shape = output_shape.to_shape(); + std::vector shape_values(static_shape.begin(), static_shape.end()); + target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape_values.size()}, shape_values); + } else { + auto input = process_view_input_new(context, 0); + target_shape = std::make_shared(input, ov::element::i64); + } + + auto res = std::make_shared(scalar, target_shape); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 18643371e329..0fe8e0a8d067 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -29,20 +31,17 @@ OutputVector translate_set_rows(const NodeContext & context) { num_inputs_check(context, 3, 3); auto data = process_view_input_new(context, 0); - auto indices = context.get_input(1); - auto dst = context.get_input(2); + auto indices = process_view_input_new(context, 1); + auto dst = process_view_input_new(context, 2); data = std::make_shared(data, context.get_output_type()); - auto row_size = context.get_input_shape(2)[3].get_length(); + const auto indices_shape = context.get_input_shape(1); + const bool multidim_indices = indices_shape.rank().is_static() && + indices_shape.rank().get_length() == 4 && + ((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) || + (indices_shape[2].is_static() && indices_shape[2].get_length() > 1)); - auto ind_squeezed = - std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); - auto data_reshaped = std::make_shared( - data, - ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), - false); auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); Output res; @@ -53,11 +52,31 @@ OutputVector translate_set_rows(const NodeContext & context) { data = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false); res = std::make_shared(OutputVector{dst, data}, concat_axis); + } else if (multidim_indices) { + auto updates_shape = std::make_shared(data, ov::element::i64); + + auto indices_rank3 = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto indices_rank4_shape = std::make_shared(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0); + auto indices_rank4 = std::make_shared(indices_rank3, indices_rank4_shape, false); + auto broadcasted_indices = std::make_shared(indices_rank4, updates_shape); + + res = std::make_shared(dst, broadcasted_indices, data, axes); } else { + auto row_size = context.get_input_shape(2)[3].get_length(); + auto ind_squeezed = std::make_shared( + indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); + auto data_reshaped = std::make_shared( + data, + ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), + false); res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); } - if (auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr())) { + auto dst_reshape = std::dynamic_pointer_cast(dst.get_node_shared_ptr()); + if (!multidim_indices && dst_reshape) { // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] // ctx_per_seq is not fixed due to llama-bench compatibility auto dst_shape_partial = dst_reshape->get_input_partial_shape(0); diff --git a/ggml/src/ggml-openvino/openvino/op/sqr.cpp b/ggml/src/ggml-openvino/openvino/op/sqr.cpp new file mode 100644 index 000000000000..9ea886e73567 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sqr.cpp @@ -0,0 +1,35 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sqr(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +OutputVector translate_sqrt(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov \ No newline at end of file diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 59fd26df8cd5..cca448a7cec1 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_CONCAT", op::translate_concat }, {"GGML_OP_CONT", op::translate_cont }, {"GGML_OP_DIV", op::translate_div }, + {"GGML_OP_FILL", op::translate_fill }, {"GGML_OP_GET_ROWS", op::translate_get_rows }, {"GGML_OP_IM2COL", op::translate_im2col }, {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, @@ -37,11 +39,14 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SUM_ROWS", op::translate_sum_rows }, {"GGML_OP_ROPE", op::translate_rope }, {"GGML_OP_SCALE", op::translate_scale }, + {"GGML_OP_SQR", op::translate_sqr }, + {"GGML_OP_SQRT", op::translate_sqrt }, {"GGML_OP_SOFT_MAX", op::translate_soft_max }, {"GGML_OP_ARGSORT", op::translate_argsort }, {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, {"GGML_OP_TRANSPOSE", op::translate_transpose }, {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index 1d695fa12588..cd35e1429ec7 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -14,6 +14,7 @@ GGML_OP_CONVERTER(translate_cont); GGML_OP_CONVERTER(translate_concat); GGML_OP_CONVERTER(translate_add_id); GGML_OP_CONVERTER(translate_div); +GGML_OP_CONVERTER(translate_fill); GGML_OP_CONVERTER(translate_get_rows); GGML_OP_CONVERTER(translate_im2col); GGML_OP_CONVERTER(translate_mulmat); @@ -24,8 +25,10 @@ GGML_OP_CONVERTER(translate_rms_norm); GGML_OP_CONVERTER(translate_norm); GGML_OP_CONVERTER(translate_l2_norm); GGML_OP_CONVERTER(translate_sum_rows); +GGML_OP_CONVERTER(translate_sqr); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); +GGML_OP_CONVERTER(translate_sqrt); GGML_OP_CONVERTER(translate_unary_silu); GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index d00c438e2a1f..4e981c5cff7d 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -216,6 +216,13 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo (*tensor_map)[output_name] = converted_outputs[i]; } } + + const auto & node_output_aliases = decoder->get_output_aliases(node_idx); + for (const auto & output_alias : node_output_aliases) { + if (!converted_outputs.empty() && converted_outputs[0].get_node_shared_ptr() != nullptr) { + (*tensor_map)[output_alias] = converted_outputs[0]; + } + } }; if (!m_naive) { diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 70af08bdf182..04d1cad90414 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -4,6 +4,7 @@ #include "ggml-openvino-extra.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" +#include "model-cache.h" #include "openvino/frontend.h" #include "openvino/input_model.h" @@ -286,13 +287,101 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { + // Fail fast: a cache-miss recompile feeds weight data to compile_model, but + // GGML_OPENVINO_RELEASE_WEIGHTS may have already dropped the host weight pages + // (they would read as zeros). That mode requires stable graph shapes. + if (ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " + "released via GGML_OPENVINO_RELEASE_WEIGHTS. This mode requires stable graph shapes; " + "unset GGML_OPENVINO_RELEASE_WEIGHTS for dynamic workloads."); + } if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache.erase(key); } bool model_is_splitted = is_model_splitted(cgraph); + // Frontend-level model cache (GGML_OPENVINO_MODEL_CACHE_DIR): if this model + // was compiled before, import the saved blob and skip requant + convert + + // compile. Only the dynamic single-model path is cached (split models compile + // two graphs and are left to the plugin-level ov::cache_dir). The decoder is + // still needed for I/O mapping, but can be built without weight nodes since + // the weights are baked into the imported CompiledModel. + const std::string model_cache_dir = ggml_openvino_model_cache_dir(); + uint64_t model_fp = 0; + std::string blob_path, manifest_path; + bool imported = false; + // When the frontend model cache is active it supersedes the plugin-level + // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot + // be re-imported (import returns an uninitialized model). Strip cache_dir / + // cache_mode from the config used for the cached compile and the import. + ov::AnyMap mc_config = config; + if (!model_cache_dir.empty()) { + mc_config.erase("CACHE_DIR"); + mc_config.erase("CACHE_MODE"); + } + if (!model_cache_dir.empty() && !model_is_splitted) { + uint64_t extra_cfg = 0; + extra_cfg = extra_cfg * 131 + (stateful ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") ? 2u : 0u); + model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, + 15, extra_cfg); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + + std::ifstream blob_in(blob_path, std::ios::binary); + bool blob_ok = blob_in.is_open(); + bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); + if (blob_ok && manifest_ok) { + int64_t import_start = ggml_time_us(); + try { + ov::CompiledModel cm; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + cm = core.import_model(blob_in, remote_context.value(), mc_config); + } else { + cm = core.import_model(blob_in, device, mc_config); + } + // Lightweight decoder: names-only weight map (membership is all the + // decoder needs; weights live in the imported model). + std::map> weight_names; + for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) { + weight_names[n] = nullptr; + } + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, + is_static, stateful, model_is_splitted); + infer_request = std::make_shared(cm.create_infer_request()); + entry->ptr = ggml_decoder; + // Names must match the decoder's ggml-tensor keys. The non-cached + // path keys off Parameter/Result *friendly names* (set by the + // frontend); export_model preserves these, and each compiled-model + // port's node is exactly that Parameter/Result. Use the port nodes + // directly (NOT get_runtime_model(), whose graph differs and is + // unsafe to deref this way). + for (const auto & p : cm.inputs()) { + ov_input_names.push_back(p.get_node()->get_friendly_name()); + } + for (const auto & o : cm.outputs()) { + ov_output_names.push_back(o.get_node()->get_friendly_name()); + } + imported = true; + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO(" - Model cache import time: %.3f ms \n", + (ggml_time_us() - import_start) / 1000.0); + } + GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str()); + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what()); + imported = false; + } + } + } + std::shared_ptr model; + if (imported) { + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + } else { auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, @@ -311,14 +400,43 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ov::serialize(model, timestamped_filename); } + // Use the cache-stripped config when the frontend model cache is active, so + // the resulting CompiledModel can be exported and later re-imported. + const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; ov::CompiledModel compiled_model; auto remote_context = ggml_openvino_get_remote_context(); if (remote_context.has_value()) { - compiled_model = core.compile_model(model, remote_context.value(), config); + compiled_model = core.compile_model(model, remote_context.value(), compile_config); } else { - compiled_model = core.compile_model(model, device, config); + compiled_model = core.compile_model(model, device, compile_config); } compile_end_time = ggml_time_us(); + + // Export to the frontend model cache for next time. Write the blob to a + // temp file then rename (atomic) so a concurrent/crashed run never sees a + // half-written blob; write the manifest first so a present blob always has + // a verifiable manifest. + if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { + try { + if (ggml_openvino_model_cache_write_manifest(manifest_path, cgraph, model_fp)) { + const std::string tmp = blob_path + ".tmp"; + std::ofstream blob_out(tmp, std::ios::binary | std::ios::trunc); + if (blob_out.is_open()) { + compiled_model.export_model(blob_out); + blob_out.close(); + if (blob_out.good()) { + std::rename(tmp.c_str(), blob_path.c_str()); + GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); + } else { + std::remove(tmp.c_str()); + } + } + } + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); + } + } + infer_request = std::make_shared(compiled_model.create_infer_request()); entry->ptr = ggml_decoder; @@ -328,6 +446,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< for (const auto & ov_output : model->get_results()) { ov_output_names.push_back(ov_output->get_friendly_name()); } + } // end non-imported (compile) path if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); @@ -390,6 +509,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } } + // GGML_OPENVINO_RELEASE_WEIGHTS: on GPU the plugin holds its own device copy of + // every weight after compile, so the host weight buffers can be dropped to reclaim + // RSS. The GPU backend uses a single dynamic-shape model for both prefill and decode, + // so once a graph is compiled it is reused for the whole session — the only thing + // that forces a recompile is clear_caches() on backend teardown. We therefore release + // on the first cache-hit (model compiled, plugin has its copy) and, crucially, pin the + // compiled-model cache so it survives backend teardown (see ggml_backend_openvino_free). + // Without the pin, a later test/context would recompile against the now-dropped pages. + // A genuinely new graph still fails fast at the cache-miss compile branch. + if (cache_hit && device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_RELEASE_WEIGHTS") && + !ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); + } + return GGML_STATUS_SUCCESS; } @@ -670,7 +803,17 @@ bool is_model_splitted(ggml_cgraph * cgraph) { } } // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, true); + // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM + // use the name-only collector (no weight extraction); otherwise keep the original + // behavior of building (naive) weight nodes and take their names. + std::set model_weights; + if (ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM")) { + model_weights = GgmlOvDecoder::collect_weight_names(cgraph); + } else { + for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { + model_weights.insert(kv.first); + } + } std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); // leaf nodes std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 0f682fd1856c..5af88af449d3 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1781,6 +1781,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.name =*/ { 0 }, /*.extra =*/ NULL, /*.padding =*/ { 0 }, + /*.org_src =*/ NULL, }; // TODO: this should not be needed as long as we don't rely on aligned SIMD loads diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index f39abe773fc6..1370b3e92567 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -512,6 +512,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg size_t max_device_label_length = 4; { std::vector devices_meta; + bool has_openvino = false; { const size_t device_count = ggml_backend_dev_count(); for (size_t i = 0; i < device_count; i++) { @@ -519,6 +520,10 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg dev_configs.emplace_back(std::vector{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER); max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length()); + if (strncmp(ggml_backend_dev_name(dev), "OPENVINO", 8) == 0) { + has_openvino = true; + } + // cpu-based devices cannot be used in tensor split mode if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) { devices_meta.push_back(dev); @@ -526,7 +531,9 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg } } - dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + if (!has_openvino) { + dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR); + } } size_t max_arch_name_length = 0;