Refactor OpenVINO backend: move and clean up parameter handling

ggml/include/ggml.h

@@ -695,7 +695,9 @@ extern "C" {
 
         void * extra; // extra things e.g. for ggml-cuda.cu
 
-        char padding[8];
+        char padding[16];
+        // add a struct ggml_tensor * named org_src, initialized to NULL, for keeping track of original source tensors in case of in-place operations
+        struct ggml_tensor * org_src;
     };
 
     static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);

ggml/src/ggml-backend.cpp

@@ -1242,6 +1242,28 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
         GGML_ASSERT(*cur_backend_id != -1);
     }
 
+    // OpenVINO currently uses ggml tensor names as graph indices. Some models (e.g. gpt-oss and
+    // llama4) can contain duplicate ggml tensor names, so we append node ids here to keep names
+    // unique. This is a temporary workaround and will be further optimized away in the future.
+    {
+        bool has_openvino_backend = false;
+        for (int i = 0; i < sched->n_backends; i++) {
+            if (strcmp(ggml_backend_name(sched->backends[i]), "OPENVINO") == 0) {
+                has_openvino_backend = true;
+                break;
+            }
+        }
+
+        if (has_openvino_backend) {
+            for (int i = 0; i < graph->n_nodes; i++) {
+                struct ggml_tensor * node = graph->nodes[i];
+                char                 new_name[128];
+                snprintf(new_name, sizeof(new_name), "%s#%d", node->name, i);
+                ggml_format_name(node, "%s", new_name);
+            }
+        }
+    }
+
     // pass 5: split graph, find tensors that need to be copied
     {
         int i_split = 0;
@@ -1360,6 +1382,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
                                 ggml_set_input(tensor_copy);
                                 ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
                             }
+                            tensor_copy->org_src = src;
                             tensor_id_copy(src_id, cur_backend_id, c) = tensor_copy;
                             SET_CAUSE(tensor_copy, "4.cpy");
                         }

ggml/src/ggml-openvino/ggml-decoder.h

@@ -59,8 +59,6 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
         std::map<std::string, ggml_tensor *> node_inputs;
         std::map<std::string, std::vector<std::pair<std::string, ggml_tensor *>>> node_inputs_views;
         std::vector<std::string> node_inputs_names;
-        ggml_tensor * node_output;
-        std::string node_output_name;
         int node_op_case = 0;
         void * data_addr;
     };
@@ -88,6 +86,22 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual std::vector<size_t> get_input_stride(int node_idx, const std::string & name) const override;
 
+    virtual ov::Shape get_input_ggml_shape(int node_idx, const std::string & name) const override;
+
+    virtual std::vector<size_t> get_input_ggml_stride(int node_idx, const std::string & name) const override;
+
+    virtual int32_t get_input_dynamic_dim(int node_idx, const std::string & name) const override;
+
+    virtual size_t get_input_type_size(int node_idx, const std::string & name) const override;
+
+    virtual int64_t get_input_block_size(int node_idx, const std::string & name) const override;
+
+    virtual bool input_has_same_shape_as_output(int node_idx, const std::string & name) const override;
+
+    virtual bool input_is_none_op(int node_idx, const std::string & name) const override;
+
+    virtual bool input_has_org_src(int node_idx, const std::string & name) const override;
+
     virtual size_t get_view_input_size(int node_idx, const std::string & name) const override;
 
     virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override;
@@ -148,6 +162,12 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual std::vector<size_t> get_output_stride(int node_idx) const override;
 
+    virtual ov::Shape get_output_ggml_shape(int node_idx) const override;
+
+    virtual std::vector<size_t> get_output_ggml_stride(int node_idx) const override;
+
+    virtual size_t get_output_type_size(int node_idx) const override;
+
     virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override;
 
     virtual int32_t * get_output_op_params(int node_idx) const override;
@@ -156,6 +176,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual std::vector<std::string> get_output_names(int node_idx) const override;
 
+    virtual std::vector<std::string> get_output_aliases(int node_idx) const override;
+
     virtual const std::string & get_op_type() const override;
 
     virtual const std::string & get_op_type(int node_idx) const override;
@@ -173,18 +195,14 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelInputInfo> & get_model_inputs() const override {
         return m_model_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_extra_inputs() const override {
+    virtual const std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> & get_model_extra_inputs() const override {
         return m_model_extra_inputs;
     }
 
-    virtual const std::map<std::string, std::shared_ptr<ov::Tensor>> & get_model_extra_input_values() const {
-        return m_model_extra_input_values;
-    }
-
     virtual const std::map<std::string, std::shared_ptr<ov::Node>> & get_model_weights() const override {
         return m_model_weights;
     }
@@ -267,7 +285,7 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     void update_io(ggml_cgraph * cgraph);
 
     inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) {
-        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE;
+        return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE && op->src[0]->org_src == nullptr;
     }
 
     inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) {
@@ -329,9 +347,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     ggml_cgraph * m_cgraph = nullptr;
     std::map<std::string, ggml_tensor *> m_inputs;
 
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_inputs;
-    std::map<std::string, std::shared_ptr<ov::Node>> m_model_extra_inputs;
-    std::map<std::string, std::shared_ptr<ov::Tensor>> m_model_extra_input_values;
+    std::map<std::string, ov::frontend::ggml::ModelInputInfo> m_model_inputs;
+    std::map<std::string, ov::frontend::ggml::ModelExtraInputInfo> m_model_extra_inputs;
     std::map<std::string, std::shared_ptr<ov::Node>> m_model_weights;
     std::map<std::string, ggml_tensor *> m_model_outputs;
     std::vector<std::string> m_model_output_names;

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -252,14 +252,24 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // Only handle 2D weight tensors
-    if (tensor->ne[2] != 1 || tensor->ne[3] != 1) {
+    // Most quantized weights use the existing 2D extraction path. MXFP4 also
+    // appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
+    if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
         return layout;
     }
 
     int64_t n_elements = ggml_nelements(tensor);
     const size_t alignment = 64;  // Good for SIMD
 
+    if (tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) {
+        layout.weights_per_block = 32;
+        layout.is_symmetric = true;
+        layout.weights_size = ggml_nbytes(tensor);
+        layout.weights_offset = 0;
+        layout.total_size = layout.weights_size;
+        return layout;
+    }
+
     // Check if requantization is needed (NPU-specific)
     auto requant_type = ggml_openvino_get_requant_type(tensor, use_bias);
     if (requant_type.has_value()) {
@@ -334,6 +344,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     layout.is_symmetric = false;
 
     switch (tensor->type) {
+    case GGML_TYPE_MXFP4:
+        layout.is_u4 = true;
+        layout.is_symmetric = true;
+        break;
+
     case GGML_TYPE_Q4_0:
         layout.is_u4 = true;
         layout.is_symmetric = true;
@@ -369,9 +384,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // Weights: U4 = n_elements/2 bytes, U8 = n_elements bytes
     layout.weights_size = layout.is_u4 ? (n_elements / 2) : n_elements;
 
-    // Scales: F16 per block
+    // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
     int64_t n_blocks = n_elements / layout.weights_per_block;
-    layout.scales_size = n_blocks * sizeof(uint16_t);  // F16 = 2 bytes
+    layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
     // For symmetric quantization, no zp needed (weights stored as signed)
     if (layout.is_symmetric) {
         layout.zp_size = 0;

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -237,8 +237,9 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
     bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr);
     // 2D tensor (typical weight shape)
     bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1);
+    bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4;
 
-    if (is_weight_buffer && is_full_tensor_set && is_2d) {
+    if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) {
         try {
             auto result = process_weight_tensor(tensor, data, tensor->data);
             result.weight_node->set_friendly_name(tensor->name);
@@ -458,8 +459,9 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff
                                                                const ggml_tensor * tensor) {
     GGML_UNUSED(buft);
 
-    // For quantized 2D tensors (weights), we need extra space for extracted data
-    if (ggml_is_quantized(tensor->type) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {
+    // For quantized weight tensors, we need extra space for extracted data.
+    if (ggml_is_quantized(tensor->type) &&
+        ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) {
         ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);
         if (layout.total_size > 0) {
             // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n",
@@ -901,17 +903,10 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
             return true;
         }
 
-        // Keep the MoE routing weights gather on CPU for GPU runs. Splitting
-        // only at the later SUM/CLAMP/DIV nodes still leaves this routing path
-        // numerically unstable for arctic-style MoE graphs.
-        if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
-            return true;
-        }
         break;
     }
     case GGML_OP_RESHAPE: {
-        if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 ||
-            strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
+        if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
             return true;
         }
         break;
@@ -938,69 +933,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         break;
     }
     case GGML_OP_DIV: {
-        bool requires_broadcast = false;
-        for (int i = 0; i < 4; i++) {
-            if (op->src[0]->ne[i] == op->src[1]->ne[i]) {
-                continue;
-            }
-
-            if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) {
-                return true;
-            }
-
-            requires_broadcast = true;
-        }
-
         // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path
         // and produce infs for per-channel scale vectors. Keep those DIVs on CPU
         // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
-        if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") {
-            return true;
-        }
-
-        // qwen3next MoE weight normalization is numerically sensitive on the GPU
-        // path. Keep the normalization divide on CPU to match the reference.
-        if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) {
-            return true;
-        }
-        break;
-    }
-    case GGML_OP_SOFT_MAX: {
-        if (op->src[2] != nullptr) {
-            // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n");
-            return true;
-        }
-
-        if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) {
-            return true;
-        }
-
-        // GPU execution of the MoE routing weights softmax is numerically unstable
-        // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax
-        // on CPU so the scheduler splits at the same boundary that restores parity.
-        if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr &&
-            strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) {
+        if (op->src[1]->ne[0] == 1 && op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 384) {
             return true;
         }
         break;
     }
     case GGML_OP_SUM_ROWS: {
-        if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) {
-            return true;
-        }
-
         // if the input is PERMUTE skip
         if (op->src[0]->op == GGML_OP_PERMUTE) {
             return true;
         }
         break;
     }
-    case GGML_OP_CLAMP: {
-        if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) {
-            return true;
-        }
-        break;
-    }
     case GGML_OP_FLASH_ATTN_EXT: {
         float scale = 1.0f;
         float max_bias = 0.0f;
@@ -1060,12 +1007,6 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         break;
     }
     case GGML_OP_MUL_MAT: {
-        if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX &&
-            op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr &&
-            op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr &&
-            op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) {
-            return true;
-        }
         if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
             return true;
         }
@@ -1075,12 +1016,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         break;
     }
     case GGML_OP_MUL_MAT_ID: {
-        if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 ||
-            strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) {
-            return true;
-        }
-
-        if (mul_mat_id_requires_large_tmp(op)) {
+        if (mul_mat_id_requires_large_tmp(op) &&
+            !(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) {
             return true;
         }
         break;
@@ -1158,8 +1095,8 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
         return true;
     }
     case GGML_OP_VIEW: {
-        // Skip TOPK_MOE fused tests until it is fully supported
-        // the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe
+        // Skip TOPK_MOE fused tests until it is fully supported.
+        // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
         if (strcmp(op->name, "selected_experts") == 0) {
             return true;
         }
@@ -1176,7 +1113,8 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
 
     static std::unordered_set<ggml_type> supported_types{
         GGML_TYPE_F32,  GGML_TYPE_F16,  GGML_TYPE_BF16, GGML_TYPE_I64,  GGML_TYPE_I32,  GGML_TYPE_Q4_0,
-        GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K};
+        GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K,
+        GGML_TYPE_MXFP4};
 
     // derive supported op sets from the op_table map, keys in
     // the map use the full macro name (e.g. "GGML_OP_ADD"), while
@@ -1274,7 +1212,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
             // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
             return false;
         }
-        if (ggml_is_quantized(src->type) && src->ne[2] != 1) {
+        const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 &&
+                                               src->type == GGML_TYPE_MXFP4;
+        if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) {
             // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
             return false;
         }