Merge pull request pytorch#11 from ynimmaga/portable_kernels

Get openvino backend device from compile specs

Author: cavusmustafa

backends/openvino/CMakeLists.txt

@@ -69,4 +69,3 @@ target_link_options(openvino_backend PRIVATE -Wl,-rpath=${OPENVINO_LIB_PATH})
 
 # Install OpenVINO backend library to the lib directory
 install(TARGETS openvino_backend DESTINATION lib)
-

backends/openvino/partitioner.py

@@ -7,6 +7,7 @@
 from typing import Callable, final, List, Optional, Tuple
 
 import torch
+import torch.fx as fx
 from executorch.backends.openvino.preprocess import OpenvinoBackend
 from executorch.exir.backend.backend_details import CompileSpec
 from executorch.exir.backend.partitioner import (
@@ -15,12 +16,12 @@
     PartitionResult,
 )
 from executorch.exir.backend.utils import tag_constant_data
+from openvino.frontend.pytorch.torchdynamo.op_support import OperatorSupport
 
 from torch.export.exported_program import ExportedProgram
 from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner
 from torch.fx.passes.operator_support import OperatorSupportBase
-import torch.fx as fx
-from openvino.frontend.pytorch.torchdynamo.op_support import OperatorSupport
+
 
 class OpenvinoOperatorsSupport(OperatorSupportBase):
 
@@ -44,10 +45,10 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool:
         options = []
         op_type = node.target.__name__
         supported_ops = OperatorSupport(options)._support_dict
-        if (op_type == "getitem"):
+        if op_type == "getitem":
             return True
 
-        if ("torch.ops." + str(op_type) in supported_ops):
+        if "torch.ops." + str(op_type) in supported_ops:
             return True
         else:
             print("Op not supported: ", "torch.ops." + str(op_type))
@@ -58,7 +59,7 @@ def is_node_supported(self, _, node: torch.fx.Node) -> bool:
             )
             return False
 
-        return False 
+        return False
 
 
 @final
@@ -88,13 +89,12 @@ def ops_to_not_decompose(
         return (ops_not_decompose, None)
 
     def partition(self, exported_program: ExportedProgram) -> PartitionResult:
-        options = {}
         gm = fx.symbolic_trace(exported_program.graph_module)
 
         partitioner = CapabilityBasedPartitioner(
             exported_program.graph_module,
             OpenvinoOperatorsSupport(self._op_types_to_skip, self._op_names_to_skip),
-            allows_single_node_partition=True
+            allows_single_node_partition=True,
         )
         partition_list = partitioner.propose_partitions()
 

backends/openvino/preprocess.py

@@ -7,7 +7,7 @@
 import contextlib
 import struct
 
-from typing import final, List, cast
+from typing import cast, final, List
 
 import torch
 from executorch.exir.backend.backend_details import (
@@ -18,8 +18,6 @@
 from executorch.exir.backend.compile_spec_schema import CompileSpec
 from openvino.frontend.pytorch.torchdynamo.compile import openvino_compile
 
-SKIP_COMPILE_SPEC_KEYS = {"ImportForever"}
-
 
 @final
 class OpenvinoBackend(BackendDetails):
@@ -34,8 +32,8 @@ def preprocess(
         output_names = edge_program.graph_signature.user_outputs
         args = []
         for node in edge_program.graph.nodes:
-            if (node.target in input_names):
-                args.append( node.meta["val"])
+            if node.target in input_names:
+                args.append(node.meta["val"])
 
         input_shapes = []
         output_shapes = []

backends/openvino/runtime/OpenvinoBackend.cpp

@@ -7,8 +7,8 @@
  */
 
 #include <cstring>
-#include <memory>
 #include <iostream>
+#include <memory>
 
 #include <openvino/openvino.hpp>
 
@@ -39,143 +39,159 @@ namespace backends {
 namespace openvino {
 
 OpenvinoBackend::OpenvinoBackend() {
-    if (!is_available()) {
-        //ET_LOG(Error, "OpenVINO runtime is not available. Initialization failed.");
-        throw std::runtime_error("OpenVINO runtime not available");
-    }
+  if (!is_available()) {
+    // ET_LOG(Error, "OpenVINO runtime is not available. Initialization
+    // failed.");
+    throw std::runtime_error("OpenVINO runtime not available");
+  }
 
-    //ET_LOG(Info, "OpenVINO runtime successfully verified and initialized.");
+  // ET_LOG(Info, "OpenVINO runtime successfully verified and initialized.");
 }
 
 bool OpenvinoBackend::is_available() const {
-    try {
-        // Create an OpenVINO Core object to verify runtime availability
-        ov::Core core;
-
-        // Check if at least one device is available
-        auto devices = core.get_available_devices();
-        if (!devices.empty()) {
-            return true;  // OpenVINO is available
-        }
-    } catch (const std::exception& e) {
-        // Log the exception if OpenVINO runtime is not available
-        ET_LOG(Error, "OpenVINO is not available: %s", e.what());
-    } catch (...) {
-        // Handle any unexpected errors
-        ET_LOG(Error, "OpenVINO availability check failed due to an unknown error.");
-    }
+  try {
+    // Create an OpenVINO Core object to verify runtime availability
+    ov::Core core;
 
-    return false;  // OpenVINO is not available
+    // Check if at least one device is available
+    auto devices = core.get_available_devices();
+    if (!devices.empty()) {
+      return true; // OpenVINO is available
+    }
+  } catch (const std::exception& e) {
+    // Log the exception if OpenVINO runtime is not available
+    ET_LOG(Error, "OpenVINO is not available: %s", e.what());
+  } catch (...) {
+    // Handle any unexpected errors
+    ET_LOG(
+        Error, "OpenVINO availability check failed due to an unknown error.");
+  }
+
+  return false; // OpenVINO is not available
 }
 
 Result<DelegateHandle*> OpenvinoBackend::init(
     BackendInitContext& context,
     FreeableBuffer* processed,
     ArrayRef<CompileSpec> compile_specs) const {
+  ET_LOG(Info, "OpenvinoBackend::init %p", processed->data());
 
-    ET_LOG(Info, "OpenvinoBackend::init %p", processed->data());
+  ov::Core core;
+  const char* data_ptr = static_cast<const char*>(processed->data());
+  size_t data_size = processed->size();
 
-    ov::Core core;
-    const char* data_ptr = static_cast<const char*>(processed->data());
-    size_t data_size = processed->size();
+  // Copy data to a string or vector
+  std::string data_string(data_ptr, data_size);
 
-    // Copy data to a string or vector
-    std::string data_string(data_ptr, data_size);
+  // Wrap the data in a stream
+  std::istringstream compiled_stream(data_string);
 
-    // Wrap the data in a stream
-    std::istringstream compiled_stream(data_string);
+  auto device = "CPU";
+  // Get the device value, if provided in compile sepcs
+  for (auto& compile_spec : compile_specs) {
+    if (std::strcmp(compile_spec.key, "device") == 0)
+      device = static_cast<char*>(compile_spec.value.buffer);
+  }
 
-    // Import the model
-    auto compiled_model = core.import_model(compiled_stream, "CPU");
+  // Import the model
+  auto compiled_model = core.import_model(compiled_stream, device);
 
-    // Allocate an infer request
-    std::shared_ptr<ov::InferRequest> infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
+  // Allocate an infer request
+  std::shared_ptr<ov::InferRequest> infer_request =
+      std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
 
-    // Allocate execution handle
-    MemoryAllocator* allocator = context.get_runtime_allocator();
-    ExecutionHandle* handle = ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle);
-    handle->compiled_model = std::make_shared<ov::CompiledModel>(compiled_model);
-    handle->infer_request = infer_request;
+  // Allocate execution handle
+  MemoryAllocator* allocator = context.get_runtime_allocator();
+  ExecutionHandle* handle =
+      ET_ALLOCATE_INSTANCE_OR_RETURN_ERROR(allocator, ExecutionHandle);
+  handle->compiled_model = std::make_shared<ov::CompiledModel>(compiled_model);
+  handle->infer_request = infer_request;
 
-    return handle;
+  return handle;
 }
 
 Error OpenvinoBackend::execute(
     BackendExecutionContext& context,
     DelegateHandle* input_handle,
     EValue** args) const {
+  ExecutionHandle* execution_handle = (ExecutionHandle*)input_handle;
 
-    ExecutionHandle* execution_handle = (ExecutionHandle*)input_handle;
+  auto infer_request = execution_handle->infer_request;
 
-    auto infer_request = execution_handle->infer_request;
+  size_t num_inputs = infer_request->get_compiled_model().inputs().size();
+  size_t num_outputs = infer_request->get_compiled_model().outputs().size();
 
-    size_t num_inputs = infer_request->get_compiled_model().inputs().size();
-    size_t num_outputs = infer_request->get_compiled_model().outputs().size();
+  // Set inputs
+  for (size_t i = 0; i < num_inputs; i++) {
+    auto input_tensor = args[i]->toTensor();
+    ov::Shape input_shape(
+        input_tensor.sizes().begin(), input_tensor.sizes().end());
 
-    // Set inputs
-    for (size_t i = 0; i < num_inputs; i++) {
-      auto input_tensor = args[i]->toTensor();
-      ov::Shape input_shape(input_tensor.sizes().begin(), input_tensor.sizes().end());
+    // Convert input tensor to OpenVINO tensor
+    ov::element::Type ov_type =
+        convert_to_openvino_type(input_tensor.scalar_type());
+    ov::Tensor ov_input_tensor(
+        ov_type, input_shape, input_tensor.mutable_data_ptr());
 
-      // Convert input tensor to OpenVINO tensor
-      ov::element::Type ov_type = convert_to_openvino_type(input_tensor.scalar_type());
-      ov::Tensor ov_input_tensor(ov_type, input_shape, input_tensor.mutable_data_ptr());
+    infer_request->set_input_tensor(i, ov_input_tensor);
+  }
 
-      infer_request->set_input_tensor(i, ov_input_tensor);
-    }
+  // Set outputs
+  for (size_t i = 0; i < num_outputs; i++) {
+    auto output_tensor = args[num_inputs + i]->toTensor();
+    ov::Shape output_shape(
+        output_tensor.sizes().begin(), output_tensor.sizes().end());
 
-    // Set outputs
-    for (size_t i = 0; i < num_outputs; i++) {
-      auto output_tensor = args[num_inputs+i]->toTensor();
-      ov::Shape output_shape(output_tensor.sizes().begin(), output_tensor.sizes().end());
+    // Convert input tensor to OpenVINO tensor
+    ov::element::Type ov_type =
+        convert_to_openvino_type(output_tensor.scalar_type());
+    ov::Tensor ov_output_tensor(
+        ov_type, output_shape, output_tensor.mutable_data_ptr());
 
-      // Convert input tensor to OpenVINO tensor
-      ov::element::Type ov_type = convert_to_openvino_type(output_tensor.scalar_type());
-      ov::Tensor ov_output_tensor(ov_type, output_shape, output_tensor.mutable_data_ptr());
-
-      infer_request->set_output_tensor(i, ov_output_tensor);
-    }
+    infer_request->set_output_tensor(i, ov_output_tensor);
+  }
 
-    // Execute the inference
-    infer_request->infer();
+  // Execute the inference
+  infer_request->infer();
 
-    return Error::Ok;
+  return Error::Ok;
 }
 
 void OpenvinoBackend::destroy(DelegateHandle* handle) const {
-    if (!handle) {
-        ET_LOG(Info, "Attempted to destroy a null handle.");
-        return;
-    }
-
-    // Cast the handle to the appropriate type
-    ExecutionHandle* execution_handle = static_cast<ExecutionHandle*>(handle);
-
-    // Clean up resources
-    if (execution_handle->infer_request) {
-        execution_handle->infer_request.reset();  // Release the infer request
-        ET_LOG(Info, "Infer request successfully destroyed.");
-    }
-
-    if (execution_handle->compiled_model) {
-        execution_handle->compiled_model.reset();  // Release the compiled model
-        ET_LOG(Info, "Compiled model successfully destroyed.");
-    }
-
-    ET_LOG(Info, "Delegate handle destroyed successfully.");
+  if (!handle) {
+    ET_LOG(Info, "Attempted to destroy a null handle.");
+    return;
+  }
+
+  // Cast the handle to the appropriate type
+  ExecutionHandle* execution_handle = static_cast<ExecutionHandle*>(handle);
+
+  // Clean up resources
+  if (execution_handle->infer_request) {
+    execution_handle->infer_request.reset(); // Release the infer request
+    ET_LOG(Info, "Infer request successfully destroyed.");
+  }
+
+  if (execution_handle->compiled_model) {
+    execution_handle->compiled_model.reset(); // Release the compiled model
+    ET_LOG(Info, "Compiled model successfully destroyed.");
+  }
+
+  ET_LOG(Info, "Delegate handle destroyed successfully.");
 }
 
-ov::element::Type OpenvinoBackend::convert_to_openvino_type(ScalarType scalar_type) const {
-    switch (scalar_type) {
-      case ScalarType::Float:
-        return ov::element::f32;
-      case ScalarType::Int:
-        return ov::element::i32;
-      case ScalarType::Char:
-        return ov::element::i8;
-      default:
-        throw std::runtime_error("Unsupported scalar type");
-    }
+ov::element::Type OpenvinoBackend::convert_to_openvino_type(
+    ScalarType scalar_type) const {
+  switch (scalar_type) {
+    case ScalarType::Float:
+      return ov::element::f32;
+    case ScalarType::Int:
+      return ov::element::i32;
+    case ScalarType::Char:
+      return ov::element::i8;
+    default:
+      throw std::runtime_error("Unsupported scalar type");
+  }
 }
 
 } // namespace openvino
@@ -185,7 +201,5 @@ ov::element::Type OpenvinoBackend::convert_to_openvino_type(ScalarType scalar_ty
 namespace {
 auto backend = executorch::backends::openvino::OpenvinoBackend();
 executorch::runtime::Backend backend_id{"OpenvinoBackend", &backend};
-static auto registered = executorch::runtime::register_backend(backend_id); 
+static auto registered = executorch::runtime::register_backend(backend_id);
 } // namespace
-
-

examples/openvino/CMakeLists.txt

@@ -94,5 +94,3 @@ set_target_properties(openvino_executor_runner PROPERTIES INSTALL_RPATH "$ORIGIN
 get_filename_component(
   EXECUTORCH_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE
 )
-
-

examples/openvino/aot/README.md

@@ -113,5 +113,3 @@ python aot_openvino_compiler.py --suite timm --model vgg16 --input_shape [1, 3,
 
 - **Unsupported Input Shape**:
   Ensure `--input_shape` is provided as a valid list or tuple.
-
-