Skip to content

Commit 8ffca64

Browse files
DannyYuyang-quicDannyYuyang-quic
committed
Qualcomm AI Engine Direct - Support Qnn IR backend in online preparation
 - Support Qnn IR backend - Replace QCir with Dlc in online prepare flow - Add config for Saver backend - Block online preparation if the QNN version is below 2.30. - Fix SDK version checking - quant/dequant op breakage fix - Upgrade ANDROID_NATIVE_API_LEVEL from 23 to 30 - Add comments for qat_training_data/passes_job
1 parent 12ed924 commit 8ffca64

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+954
-111
lines changed

‎.ci/scripts/build-qnn-sdk.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ set_up_aot() {
3333
cmake .. \
3434
-DCMAKE_INSTALL_PREFIX=$PWD \
3535
-DEXECUTORCH_BUILD_QNN=ON \
36+
-DANDROID_NATIVE_API_LEVEL=30 \
3637
-DQNN_SDK_ROOT=${QNN_SDK_ROOT} \
3738
-DEXECUTORCH_BUILD_DEVTOOLS=ON \
3839
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \

‎backends/qualcomm/CMakeLists.txt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ endif()
7070

7171
include_directories(
7272
BEFORE ${_common_include_directories} ${QNN_SDK_ROOT}/include/QNN
73+
${QNN_SDK_ROOT}/share/QNN/converter/jni
7374
${EXECUTORCH_SOURCE_DIR}/third-party/flatbuffers/include
7475
${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10
7576
)
@@ -117,6 +118,7 @@ add_library(qnn_backend STATIC)
117118
add_library(qnn_backend_cache STATIC)
118119
add_library(qnn_context STATIC)
119120
add_library(qnn_custom_protocol STATIC)
121+
add_library(qnn_dlc_manager STATIC)
120122
add_library(qnn_device STATIC)
121123
add_library(qnn_executorch_backend SHARED)
122124
add_library(qnn_executorch_header INTERFACE)
@@ -174,8 +176,11 @@ target_link_libraries(
174176
qnn_factory PRIVATE qnn_schema qnn_backend qnn_device qnn_context qnn_graph
175177
qnn_mem_manager qnn_custom_protocol
176178
)
179+
180+
target_link_libraries(qnn_dlc_manager PRIVATE qnn_factory qnn_backend qnn_device qnn_context qnn_graph qnn_mem_manager)
181+
177182
target_link_libraries(
178-
qnn_manager PRIVATE qnn_factory wrappers qnn_schema utils shared_buffer
183+
qnn_manager PRIVATE qnn_factory wrappers qnn_schema utils shared_buffer qnn_dlc_manager
179184
)
180185
target_link_libraries(
181186
qnn_executorch_backend PRIVATE qnn_executorch_header qnn_schema qnn_manager

‎backends/qualcomm/aot/python/PyQnnManagerAdaptor.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ class PyQnnManager {
195195
std::vector<std::shared_ptr<OpWrapper>>& op_wrappers) {
196196
QnnExecuTorchContextBinary binary_info;
197197

198-
if (qnn_manager_->IsOnlinePrepare() || qnn_manager_->IsMultipleGraphs()) {
198+
if (qnn_manager_->IsMultipleGraphs()) {
199199
builder_.Reset();
200200
std::vector<uint8_t> tensor_data;
201201
std::vector<uint64_t> offsets;
@@ -305,8 +305,11 @@ class PyQnnManager {
305305
QNN_EXECUTORCH_LOG_ERROR("Fail to compile QNN graph");
306306
return py::array_t<char>(0);
307307
}
308-
if (qnn_manager_->GetContextBinary(binary_info) !=
309-
executorch::runtime::Error::Ok) {
308+
auto qnn_executorch_options = GetQnnExecuTorchOptions(
309+
qnn_executorch_option_ptr_.cast<std::string_view>().data());
310+
if (qnn_executorch_options->saver() ||
311+
qnn_manager_->GetContextBinary(binary_info) !=
312+
executorch::runtime::Error::Ok) {
310313
return py::array_t<char>(0);
311314
}
312315
}

‎backends/qualcomm/builders/op_dequantize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def define_node(
4545
dequant_output_tensors = [output_tensor_wrapper]
4646

4747
dequant_op = PyQnnWrapper.PyQnnOpWrapper(
48-
node.target.__name__,
48+
node.name,
4949
QNN_OP_PACKAGE_NAME_QTI_AISW,
5050
OpDequantize.op_name,
5151
)

‎backends/qualcomm/builders/op_quantize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def define_node(
5252
quant_output_tensors = [output_tensor_wrapper]
5353

5454
quant_op = PyQnnWrapper.PyQnnOpWrapper(
55-
node.target.__name__,
55+
node.name,
5656
QNN_OP_PACKAGE_NAME_QTI_AISW,
5757
OpQuantize.op_name,
5858
)

‎backends/qualcomm/qnn_preprocess.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
from executorch.backends.qualcomm.builders.node_visitor import get_node_visitors
1616
from executorch.backends.qualcomm.builders.qnn_constants import OpContextLoader
1717
from executorch.backends.qualcomm.partition.utils import generate_qnn_executorch_option
18+
from executorch.backends.qualcomm.serialization.qc_schema_serialize import (
19+
flatbuffer_to_option,
20+
)
1821
from executorch.exir.backend.backend_details import (
1922
BackendDetails,
2023
CompileSpec,
@@ -92,6 +95,12 @@ def preprocess(
9295
qnn_manager.GetGraphNames()[0],
9396
[py_op_wrapper.GetOpWrapper() for py_op_wrapper in py_op_wrapper_list],
9497
)
98+
99+
obj_options = flatbuffer_to_option(option)
100+
if obj_options.saver:
101+
exit(
102+
f"Record all QNN API calls from saver backend at: {obj_options.saver_output_dir}"
103+
)
95104
assert len(qnn_context_binary) != 0, "Failed to generate Qnn context binary."
96105
qnn_manager.Destroy()
97106
# For now, debug_handle_map is not used by QNN ExecuTorch

‎backends/qualcomm/runtime/QnnExecuTorchBackend.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
3636
// covert SizedBuffer to qnn ExecuTorch option
3737
QnnExecuTorchContextBinary qnn_context_blob;
3838
const qnn_delegate::QnnExecuTorchOptions* qnn_executorch_options = nullptr;
39-
4039
auto [status, signature, ctx_size, ctx_bin] =
4140
QnnContextCustomProtocol().DeserializeContextCustomBuffer(
4241
const_cast<void*>(processed->data()));
@@ -74,7 +73,6 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
7473
// NOTE: Since we use placement new and since this type is not trivially
7574
// destructible, we must call the destructor manually in destroy().
7675
new (qnn_manager) QnnManager(qnn_executorch_options, qnn_context_blob);
77-
7876
// TODO: this is a temporal solution for multi-graph support, will be
7977
// removed once framework starts to accept runtime configuration
8078
// ---
@@ -96,9 +94,9 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
9694

9795
if (qnn_manager->IsOnlinePrepare()) {
9896
ET_CHECK_OR_RETURN_ERROR(
99-
qnn_manager->CompileQcir() == Error::Ok,
97+
qnn_manager->CompileDlc() == Error::Ok,
10098
Internal,
101-
"Fail to compile binary in qcir format");
99+
"Fail to compile binary in Dlc format");
102100
} else {
103101
for (const std::string& graph_name : qnn_manager->GetGraphNames()) {
104102
ET_CHECK_OR_RETURN_ERROR(

‎backends/qualcomm/runtime/QnnManager.cpp

Lines changed: 96 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,7 @@ bool CompareExportedInput(
3737
}
3838

3939
QnnManager::~QnnManager() {
40-
backend_params_ptr_.reset(new BackendConfigParameters());
41-
logger_.reset();
42-
qnn_loaded_backend_.TerminateAllBackends();
40+
Destroy();
4341
}
4442

4543
QnnManager::QnnManager(
@@ -96,10 +94,14 @@ QnnManager::QnnManager(
9694
}
9795
qnn_loaded_backend_ = QnnImplementation(library_path);
9896
backend_params_ptr_ = std::make_unique<BackendConfigParameters>();
97+
98+
qnn_dlc_manager_ =
99+
std::make_shared<QnnDlcManager>(qnn_context_blob_, options_);
99100
}
100101

101102
Error QnnManager::LoadQnnLibrary() {
102-
Error ret = qnn_loaded_backend_.Load(nullptr);
103+
auto config = GetImplementationConfig();
104+
Error ret = qnn_loaded_backend_.Load(config.get());
103105
return ret;
104106
}
105107

@@ -286,7 +288,11 @@ Error QnnManager::Init() {
286288
"parameters for Qnn executorch backend type %d",
287289
options_->backend_options()->backend_type());
288290
backend_params_ptr_ = QnnBackendFactory().Create(
289-
qnn_loaded_backend_, logger_.get(), qnn_context_blob_, options_);
291+
qnn_loaded_backend_,
292+
logger_.get(),
293+
qnn_context_blob_,
294+
options_,
295+
qnn_dlc_manager_.get());
290296
ET_CHECK_OR_RETURN_ERROR(
291297
backend_params_ptr_ != nullptr,
292298
Internal,
@@ -326,6 +332,18 @@ Error QnnManager::Init() {
326332
Internal,
327333
"Fail to pre register custom memory handle");
328334
#endif
335+
336+
if (IsOnlinePrepare()) {
337+
Qnn_ApiVersion_t qnn_version = {QNN_VERSION_INIT};
338+
qnn_loaded_backend_.GetQnnInterface().qnn_backend_get_api_version(
339+
&qnn_version);
340+
341+
ET_CHECK_OR_RETURN_ERROR(
342+
qnn_dlc_manager_->SetUpDlcEnvironment(qnn_version.coreApiVersion) ==
343+
Error::Ok,
344+
Internal,
345+
"Fail to setup Dlc environment");
346+
}
329347
return Error::Ok;
330348
}
331349

@@ -446,9 +464,11 @@ Error QnnManager::ProfileExecuteData(
446464
void QnnManager::Destroy() {
447465
QNN_EXECUTORCH_LOG_INFO("Destroy Qnn backend parameters");
448466
backend_params_ptr_.reset(new BackendConfigParameters());
467+
qnn_dlc_manager_->ResetBackendParams();
449468
logger_.reset();
450-
469+
qnn_dlc_manager_->ResetLogger();
451470
qnn_loaded_backend_.TerminateAllBackends();
471+
qnn_dlc_manager_->TerminateAllBackends();
452472
}
453473

454474
bool QnnManager::IsNodeSupportedByBackend(
@@ -483,11 +503,64 @@ bool QnnManager::IsNodeSupportedByBackend(
483503

484504
Error QnnManager::GetContextBinary(
485505
QnnExecuTorchContextBinary& qnn_executorch_context_binary) {
486-
ET_CHECK_OR_RETURN_ERROR(
487-
backend_params_ptr_->qnn_context_ptr_->GetContextBinary(
488-
qnn_executorch_context_binary) == Error::Ok,
489-
Internal,
490-
"Fail to get context binary.");
506+
if (IsOnlinePrepare() &&
507+
qnn_dlc_manager_->backend_params_ptr_->qnn_context_ptr_.get() !=
508+
nullptr) {
509+
ET_CHECK_OR_RETURN_ERROR(
510+
qnn_dlc_manager_->backend_params_ptr_->qnn_context_ptr_
511+
->GetContextBinary(qnn_executorch_context_binary) == Error::Ok,
512+
Internal,
513+
"Fail to get context binary.");
514+
}
515+
516+
else {
517+
ET_CHECK_OR_RETURN_ERROR(
518+
backend_params_ptr_->qnn_context_ptr_->GetContextBinary(
519+
qnn_executorch_context_binary) == Error::Ok,
520+
Internal,
521+
"Fail to get context binary.");
522+
}
523+
return Error::Ok;
524+
}
525+
526+
Error QnnManager::CompileDlc() {
527+
Qnn_ErrorHandle_t error;
528+
auto qnn_dlc_graph_info = qnn_dlc_manager_->GetQnnDlcGraphInfoPtr();
529+
uint32_t qnn_dlc_graph_info_num = qnn_dlc_manager_->GetQnnDlcGraphInfoNum();
530+
for (uint32_t i = 0; i < qnn_dlc_graph_info_num; ++i) {
531+
auto& graphInfo = (*qnn_dlc_graph_info)[i];
532+
backend_params_ptr_->qnn_graph_ptr_->SetGraphHandle(
533+
graphInfo.graphName, graphInfo.graph);
534+
error =
535+
backend_params_ptr_->qnn_graph_ptr_->GraphFinalize(graphInfo.graphName);
536+
if (error != QNN_SUCCESS) {
537+
QNN_EXECUTORCH_LOG_ERROR(
538+
"Failed to finalize Qnn Graph with error: %d",
539+
QNN_GET_ERROR_CODE(error));
540+
return Error::Internal;
541+
}
542+
543+
std::vector<std::shared_ptr<TensorWrapper>> graph_inputs, graph_outputs,
544+
tensors;
545+
546+
for (int i = 0; i < graphInfo.numInputTensors; ++i) {
547+
auto tw = CreateTensorWrapper(graphInfo.inputTensors[i]);
548+
tw->UpdateQnnTensorMeta(graphInfo.inputTensors[i]);
549+
graph_inputs.push_back(tw);
550+
}
551+
for (int i = 0; i < graphInfo.numOutputTensors; ++i) {
552+
auto tw = CreateTensorWrapper(graphInfo.outputTensors[i]);
553+
tw->UpdateQnnTensorMeta(graphInfo.outputTensors[i]);
554+
graph_outputs.push_back(tw);
555+
}
556+
557+
ET_CHECK_OR_RETURN_ERROR(
558+
AllocateTensor(graphInfo.graphName, graph_inputs, graph_outputs) ==
559+
Error::Ok,
560+
Internal,
561+
"Fail to allocate tensor for Dlc with graph_name: %s",
562+
graphInfo.graphName);
563+
}
491564

492565
return Error::Ok;
493566
}
@@ -616,31 +689,34 @@ Error QnnManager::Compile(
616689
const std::string& graph_name,
617690
std::vector<std::shared_ptr<OpWrapper>>& op_wrappers) {
618691
Qnn_ErrorHandle_t error = QNN_SUCCESS;
692+
QnnGraph* qnn_graph_ptr = backend_params_ptr_->qnn_graph_ptr_.get();
619693

694+
if (IsOnlinePrepare() &&
695+
qnn_dlc_manager_->backend_params_ptr_->qnn_graph_ptr_.get() != nullptr) {
696+
qnn_graph_ptr = qnn_dlc_manager_->backend_params_ptr_->qnn_graph_ptr_.get();
697+
}
620698
for (std::shared_ptr<OpWrapper>& op_wrapper : op_wrappers) {
621699
for (const auto& tensor_wrapper : op_wrapper->GetInputTensors()) {
622700
ET_CHECK_OR_RETURN_ERROR(
623-
backend_params_ptr_->qnn_graph_ptr_->EnsureTensorInQnnGraph(
624-
graph_name, tensor_wrapper) == Error::Ok,
701+
qnn_graph_ptr->EnsureTensorInQnnGraph(graph_name, tensor_wrapper) ==
702+
Error::Ok,
625703
Internal,
626704
"Tensor name %s isn't added to Qnn Graph",
627705
tensor_wrapper->GetName().c_str());
628706
}
629-
630707
for (const auto& tensor_wrapper : op_wrapper->GetOutputTensors()) {
631708
ET_CHECK_OR_RETURN_ERROR(
632-
backend_params_ptr_->qnn_graph_ptr_->EnsureTensorInQnnGraph(
633-
graph_name, tensor_wrapper) == Error::Ok,
709+
qnn_graph_ptr->EnsureTensorInQnnGraph(graph_name, tensor_wrapper) ==
710+
Error::Ok,
634711
Internal,
635712
"Tensor name %s isn't added to Qnn Graph",
636713
tensor_wrapper->GetName().c_str());
637714
}
638-
639715
for (const auto& param : op_wrapper->GetParams()) {
640716
auto* p_tensor_param = dynamic_cast<TensorParamWrapper*>(param.get());
641717
if (p_tensor_param != nullptr) {
642718
ET_CHECK_OR_RETURN_ERROR(
643-
backend_params_ptr_->qnn_graph_ptr_->EnsureTensorInQnnGraph(
719+
qnn_graph_ptr->EnsureTensorInQnnGraph(
644720
graph_name, p_tensor_param->GetTensorWrapper()) == Error::Ok,
645721
Internal,
646722
"Param tensor name %s isn't added to Qnn Graph",
@@ -652,23 +728,21 @@ Error QnnManager::Compile(
652728
"Fail to configure Qnn backend");
653729
}
654730

655-
error = backend_params_ptr_->qnn_graph_ptr_->GraphAddNode(
656-
graph_name, op_wrapper->GetOpConfig());
731+
error = qnn_graph_ptr->GraphAddNode(graph_name, op_wrapper->GetOpConfig());
657732
if (error != QNN_SUCCESS) {
658733
QNN_EXECUTORCH_LOG_ERROR(
659734
"Failed to add node to Qnn Graph with error: %d",
660735
QNN_GET_ERROR_CODE(error));
661736
return Error::Internal;
662737
}
663738
}
664-
error = backend_params_ptr_->qnn_graph_ptr_->GraphFinalize(graph_name);
739+
error = qnn_graph_ptr->GraphFinalize(graph_name);
665740
if (error != QNN_SUCCESS) {
666741
QNN_EXECUTORCH_LOG_ERROR(
667742
"Failed to finalize Qnn Graph with error: %d",
668743
QNN_GET_ERROR_CODE(error));
669744
return Error::Internal;
670745
}
671-
672746
return Error::Ok;
673747
}
674748

‎backends/qualcomm/runtime/QnnManager.h

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <executorch/backends/qualcomm/runtime/Logging.h>
1414
#include <executorch/backends/qualcomm/runtime/QnnExecuTorch.h>
1515
#include <executorch/backends/qualcomm/runtime/backends/QnnBackendFactory.h>
16+
#include <executorch/backends/qualcomm/runtime/backends/QnnDlcManager.h>
1617
#include <executorch/runtime/core/error.h>
1718

1819
#include <memory>
@@ -71,7 +72,7 @@ class QnnManager {
7172
QnnExecuTorchContextBinary& qnn_executorch_context_binary);
7273

7374
executorch::runtime::Error CompileQcir();
74-
75+
executorch::runtime::Error CompileDlc();
7576
executorch::runtime::Error Compile(
7677
const std::string& graph_name,
7778
std::vector<std::shared_ptr<OpWrapper>>& op_wrappers);
@@ -110,6 +111,22 @@ class QnnManager {
110111
std::string GetBinarySignature();
111112

112113
private:
114+
std::unique_ptr<const QnnSaver_Config_t*[]> GetImplementationConfig() {
115+
if (options_->saver()) {
116+
auto outputDirCfg = std::make_unique<QnnSaver_Config_t>();
117+
outputDirCfg->option = QNN_SAVER_CONFIG_OPTION_OUTPUT_DIRECTORY;
118+
outputDirCfg->outputDirectory = options_->saver_output_dir()->c_str();
119+
120+
auto saverCfg = std::make_unique<const QnnSaver_Config_t*[]>(2);
121+
saverCfg[0] = outputDirCfg.release();
122+
saverCfg[1] = nullptr;
123+
124+
return saverCfg;
125+
} else {
126+
return nullptr;
127+
}
128+
}
129+
113130
executorch::runtime::Error LoadQnnLibrary();
114131

115132
static constexpr const char* htp_library_name_ = "libQnnHtp.so";
@@ -147,6 +164,9 @@ class QnnManager {
147164
{Qnn_DataType_t::QNN_DATATYPE_UFIXED_POINT_16,
148165
executorch::aten::ScalarType::UInt16},
149166
};
167+
168+
// Manager for handling DLC (Deep Learning Container)
169+
std::shared_ptr<QnnDlcManager> qnn_dlc_manager_;
150170
};
151171
} // namespace qnn
152172
} // namespace backends

0 commit comments

Comments
 (0)