OpenVINO backend: GPU compile-time memory optimizations

ggml/include/ggml.h

@@ -695,7 +695,9 @@ extern "C" {
 
         void * extra; // extra things e.g. for ggml-cuda.cu
 
-        char padding[8];
+        char padding[16];
+        // add a struct ggml_tensor * named org_src, initialized to NULL, for keeping track of original source tensors in case of in-place operations
+        struct ggml_tensor * org_src;
     };
 
     static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);

ggml/src/ggml-backend.cpp

@@ -1242,6 +1242,28 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
         GGML_ASSERT(*cur_backend_id != -1);
     }
 
+    // OpenVINO currently uses ggml tensor names as graph indices. Some models (e.g. gpt-oss and
+    // llama4) can contain duplicate ggml tensor names, so we append node ids here to keep names
+    // unique. This is a temporary workaround and will be further optimized away in the future.
+    {
+        bool has_openvino_backend = false;
+        for (int i = 0; i < sched->n_backends; i++) {
+            if (strcmp(ggml_backend_name(sched->backends[i]), "OPENVINO") == 0) {
+                has_openvino_backend = true;
+                break;
+            }
+        }
+
+        if (has_openvino_backend) {
+            for (int i = 0; i < graph->n_nodes; i++) {
+                struct ggml_tensor * node = graph->nodes[i];
+                char                 new_name[128];
+                snprintf(new_name, sizeof(new_name), "%s#%d", node->name, i);
+                ggml_format_name(node, "%s", new_name);
+            }
+        }
+    }
+
     // pass 5: split graph, find tensors that need to be copied
     {
         int i_split = 0;
@@ -1360,6 +1382,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                                 ggml_set_input(tensor_copy);
                                 ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
                             }
+                            tensor_copy->org_src = src;
                             tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;
                             SET_CAUSE(tensor_copy, "4.cpy");
                         }

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -16,6 +16,7 @@
 #include <iomanip>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <openvino/core/dimension.hpp>
 #include <openvino/core/except.hpp>
 #include <openvino/core/node.hpp>
@@ -31,6 +32,7 @@
 #include <set>
 #include <stdexcept>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,
@@ -106,14 +108,6 @@ void GgmlOvDecoder::set_input_output() {
         auto node_name = std::string(node->name);
         auto node_output_name = node_name;
         auto * node_output = node;
-        if (node->op == GGML_OP_SET_ROWS) {
-            // SET_ROWS updates the tensor in place. For later ov op that uses the
-            // the view_src of SET_ROWS, we need to make sure they get the updated tensor
-            // by putting the view_src name in the tensor_map in
-            // <openvino>/src/frontends/ggml/src/translate_session.cpp
-            node_output_name = std::string(node->view_src->name);
-            node_output = node->view_src;
-        }
 
         current_node_info.node = node;
         current_node_info.node_name = node_name;
@@ -787,6 +781,42 @@ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_no
     return model_weights;
 }
 
+// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the
+// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such
+// tensors have no OV buffer context to own a cached extra, so without this they are
+// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB
+// F32 dequant each time. Keyed by tensor->data, which is stable for the process and
+// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the
+// per-tensor extra cache and never reach here.
+static std::mutex g_nonov_weight_cache_mutex;
+static std::unordered_map<const void *, std::shared_ptr<ov::Node>> g_nonov_weight_cache;
+
+std::set<std::string> GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) {
+    // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes,
+    // so topology checks don't trigger weight extraction/requantization.
+    std::set<std::string> names;
+    for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) {
+        auto * node = cgraph->nodes[node_i];
+        for (int i = 0; i < GGML_MAX_SRC; i++) {
+            auto * src = node->src[i];
+            if (src == nullptr) {
+                continue;
+            }
+            std::string src_name(src->name);
+            if (is_rope_freqs_weight(src, node)) {
+                src_name = "rope_freqs.weight";
+            }
+            if (!src->view_src) {
+                ggml_backend_buffer * buffer = src->buffer;
+                if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
+                    names.insert(src_name);
+                }
+            }
+        }
+    }
+    return names;
+}
+
 std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) {
     const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer);
 
@@ -826,6 +856,21 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer
+    // context to cache an extra in, so memoize them here keyed by their (stable) data
+    // pointer to avoid re-extracting on every recompile. Opt-in via
+    // GGML_OPENVINO_REDUCE_COMPILE_MEM. Skip for `naive` (test/naive path) since use_bias
+    // changes the produced node.
+    const bool cacheable_nonov = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_ov_buffer &&
+                                 !naive && tensor->data != nullptr;
+    if (cacheable_nonov) {
+        std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+        auto it = g_nonov_weight_cache.find(tensor->data);
+        if (it != g_nonov_weight_cache.end()) {
+            return it->second;
+        }
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -834,7 +879,7 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
     // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name);
     static const std::set<ggml_type> weight_types = {GGML_TYPE_F32,  GGML_TYPE_F16,  GGML_TYPE_BF16, GGML_TYPE_Q8_0,
                                                      GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K,
-                                                     GGML_TYPE_Q5_K, GGML_TYPE_Q6_K};
+                                                     GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4};
     if (weight_types.find(tensor->type) == weight_types.end()) {
         throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " +
                                  ggml_type_name(tensor->type));
@@ -863,6 +908,12 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
 
     ov_weight.weight_node->set_friendly_name(tensor->name);
     if (!is_ov_buffer) {
+        if (cacheable_nonov) {
+            std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+            // Another thread may have inserted concurrently; keep the first.
+            auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node);
+            return it->second;
+        }
         return ov_weight.weight_node;
     }
 
@@ -1231,6 +1282,14 @@ std::vector<std::string> GgmlOvDecoder::get_output_names(int node_idx) const {
     return {m_node_info_list[node_idx].node_output_name};
 }
 
+std::vector<std::string> GgmlOvDecoder::get_output_aliases(int node_idx) const {
+    const auto * node = m_node_info_list[node_idx].node;
+    if (node != nullptr && node->op == GGML_OP_SET_ROWS && node->view_src != nullptr) {
+        return {std::string(node->view_src->name)};
+    }
+    return {};
+}
+
 const std::string & GgmlOvDecoder::get_op_name() const {
     static const std::string unknown_name = "UNKNOWN_OP_NAME";
     return unknown_name;
@@ -1307,10 +1366,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
             if (src == nullptr) {
                 continue;
             }
-            struct ggml_tensor * root_src = nullptr;
-            // if (src->org_src) {
-            //     root_src = src->org_src;
-            // }
+            ggml_tensor * root_src = nullptr;
+            if (src->org_src) {
+                root_src = src->org_src;
+            }
             if (root_src) {
                 if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) {
                     m_node_dynamic_dims[root_src] = 0;
@@ -1388,7 +1447,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
             // identifies the dynamic dim even when two dims share the same size.
             m_node_dynamic_dims[node] = -1;
             if (m_node_dynamic_dims[node->src[0]] != -1) {
-                if (node->src[0]->op == GGML_OP_NONE) {
+                if (node->src[0]->op == GGML_OP_NONE && node->src[0]->org_src == nullptr) {
                     m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
                     break;
                 }

ggml/src/ggml-openvino/ggml-decoder.h

@@ -11,6 +11,8 @@
 #include <memory>
 #include <openvino/core/partial_shape.hpp>
 #include <optional>
+#include <set>
+#include <string>
 #include <vector>
 
 struct ModelParams {
@@ -156,6 +158,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual std::vector<std::string> get_output_names(int node_idx) const override;
 
+    virtual std::vector<std::string> get_output_aliases(int node_idx) const override;
+
     virtual const std::string & get_op_type() const override;
 
     virtual const std::string & get_op_type(int node_idx) const override;
@@ -235,6 +239,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(ggml_cgraph * cgraph,
                                                                                 bool naive = false);
 
+    // Collect just the set of weight-tensor names referenced by the graph, without
+    // building (or requantizing) any OV weight nodes. Used by topology checks like
+    // is_model_splitted that only need name membership.
+    static std::set<std::string> collect_weight_names(ggml_cgraph * cgraph);
+
     const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;
 
     const ggml_tensor * get_tensor_from_name(const std::string & name) const;
@@ -267,7 +276,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     void update_io(ggml_cgraph * cgraph);
 
     inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
-        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;
+        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE && op->src[0]->org_src == nullptr;
     }
 
     inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -45,6 +45,9 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_DISABLE_CACHE",
         "GGML_OPENVINO_DISABLE_KV_SLICE",
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
+        "GGML_OPENVINO_RELEASE_WEIGHTS",
+        "GGML_OPENVINO_REDUCE_COMPILE_MEM",
+        "GGML_OPENVINO_MODEL_CACHE_DIR",
     };
 
     for (const char * const & env_var : env_var_names) {
@@ -252,14 +255,24 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // Only handle 2D weight tensors
-    if (tensor->ne[2] != 1 || tensor->ne[3] != 1) {
+    // Most quantized weights use the existing 2D extraction path. MXFP4 also
+    // appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
+    if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
         return layout;
     }
 
     int64_t n_elements = ggml_nelements(tensor);
     const size_t alignment = 64;  // Good for SIMD
 
+    if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) {
+        layout.weights_per_block = 32;
+        layout.is_symmetric = true;
+        layout.weights_size = ggml_nbytes(tensor);
+        layout.weights_offset = 0;
+        layout.total_size = layout.weights_size;
+        return layout;
+    }
+
     // Check if requantization is needed (NPU-specific)
     auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias);
     if (requant_type.has_value()) {
@@ -334,6 +347,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     layout.is_symmetric = false;
 
     switch (tensor->type) {
+    case GGML_TYPE_MXFP4:
+        layout.is_u4 = true;
+        layout.is_symmetric = true;
+        break;
+
     case GGML_TYPE_Q4_0:
         layout.is_u4 = true;
         layout.is_symmetric = true;
@@ -369,9 +387,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes
     layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;
 
-    // Scales: F16 per block
+    // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
     int64_t n_blocks = n_elements / layout.weights_per_block;
-    layout.scales_size = n_blocks * sizeof(uint16_t);  // F16 = 2 bytes
+    layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
     // For symmetric quantization, no zp needed (weights stored as signed)
     if (layout.is_symmetric) {
         layout.zp_size = 0;

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -99,6 +99,14 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0);
 // Check if running on NPU
 bool ggml_openvino_is_npu();
 
+// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only).
+// register: record a host weight buffer (idempotent per data pointer).
+// release:  madvise(MADV_DONTNEED) all registered buffers, dropping their RSS.
+// released: true once release has run (used to fail-fast on post-release recompile).
+void ggml_openvino_register_weight_buffer(void * data, size_t size);
+void ggml_openvino_release_weight_buffers();
+bool ggml_openvino_weight_buffers_released();
+
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);