Skip to content

Fix QNN online-prepare graph handle retrieval#21860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sylvesterkaczmarek wants to merge 1 commit into
base:main
Choose a base branch
from
+32 −12
Open

Fix QNN online-prepare graph handle retrieval #21860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/test-backend-qnn.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,8 +23,8 @@ jobs:
backend: qnn
flows: >-
${{ github.event_name == 'pull_request'
&& '["qnn"]'
|| '["qnn", "qnn_16a16w", "qnn_16a8w", "qnn_16a4w", "qnn_16a4w_block", "qnn_8a8w"]' }}
&& '["qnn", "qnn_online_prepare"]'
|| '["qnn", "qnn_online_prepare", "qnn_16a16w", "qnn_16a8w", "qnn_16a4w", "qnn_16a4w_block", "qnn_8a8w"]' }}
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 120
run-linux: true
Expand Down
31 changes: 21 additions & 10 deletions backends/qualcomm/runtime/backends/QnnGraphCommon.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
/*
* Copyright (c) Qualcomm Innovation Center, Inc.
* All rights reserved.
Expand DownExpand Up@@ -28,9 +28,15 @@
return Error::Ok;
}

const auto cache_state = context_->GetCacheState();
Qnn_GraphHandle_t graph_handle = nullptr;
if (context_->GetCacheState() == QnnBackendCache::DESERIALIZE) {
// retrieve QNN Graph
if (
cache_state == QnnBackendCache::DESERIALIZE ||
cache_state == QnnBackendCache::ONLINE_PREPARE) {
// DESERIALIZE restores graphs from a context binary. ONLINE_PREPARE
// registers graphs from DLC in QnnContext::Configure before reaching here.
// In both cases the graph already exists in the context and must be
// retrieved instead of created.
error = qnn_interface.qnn_graph_retrieve(
context_->GetHandle(), graph_name.c_str(), &graph_handle);
if (error != QNN_SUCCESS) {
Expand All@@ -42,8 +48,8 @@
return Error::Internal;
}
} else if (
context_->GetCacheState() == QnnBackendCache::SERIALIZE ||
context_->GetCacheState() == QnnBackendCache::MULTI_GRAPH) {
cache_state == QnnBackendCache::SERIALIZE ||
cache_state == QnnBackendCache::MULTI_GRAPH) {
error = qnn_interface.qnn_graph_create(
context_->GetHandle(),
graph_name.c_str(),
Expand All@@ -55,27 +61,32 @@
"qnn_graph_create failed. Error %d", QNN_GET_ERROR_CODE(error));
return Error::Internal;
}
} else if (context_->GetCacheState() == QnnBackendCache::ONLINE_PREPARE) {
QNN_EXECUTORCH_LOG_INFO(
"Skip qnn_graph_create, graph has already been composed from Dlc.");
} else {
QNN_EXECUTORCH_LOG_ERROR("QNN context cache is invalid.");
return Error::Internal;
}

ET_CHECK_OR_RETURN_ERROR(
graph_handle != nullptr,
Internal,
"QNN graph handle is null after configuring graph %s.",
graph_name.c_str());

// book keep valid handle of created graph
handle_[graph_name] = graph_handle;
// The profiler needs to be created after the backend is created.
profile_[graph_name] =
std::make_unique<QnnProfile>(implementation_, backend_, profile_level_);
if (context_->GetCacheState() == QnnBackendCache::DESERIALIZE) {
if (
cache_state == QnnBackendCache::DESERIALIZE ||
cache_state == QnnBackendCache::ONLINE_PREPARE) {
ET_CHECK_OR_RETURN_ERROR(
AfterRetrieveGraph(graph_name) == Error::Ok,
Internal,
"Fail to configure after retrieving the graph.");
} else if (
context_->GetCacheState() == QnnBackendCache::SERIALIZE ||
context_->GetCacheState() == QnnBackendCache::MULTI_GRAPH) {
cache_state == QnnBackendCache::SERIALIZE ||
cache_state == QnnBackendCache::MULTI_GRAPH) {
ET_CHECK_OR_RETURN_ERROR(
AfterCreateGraph(graph_name) == Error::Ok,
Internal,
Expand Down
4 changes: 4 additions & 0 deletions backends/qualcomm/tests/tester.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -59,13 +59,15 @@ def __init__(
edge_compile_config: Optional[EdgeCompileConfig] = None,
soc_model: str = "SM8650",
use_fp16: bool = True,
online_prepare: bool = False,
):
backend_options = generate_htp_compiler_spec(use_fp16=use_fp16)
self.backend_type = backend_options.backend_type
self.chipset = get_soc_to_chipset_map()[soc_model]
self.compiler_specs = generate_qnn_executorch_compiler_spec(
soc_model=self.chipset,
backend_options=backend_options,
online_prepare=online_prepare,
)

super().__init__(
Expand DownExpand Up@@ -98,9 +100,11 @@ def __init__(
example_inputs: Tuple[torch.Tensor],
dynamic_shapes: Optional[Tuple[Any]] = None,
use_fp16: bool = True,
online_prepare: bool = False,
):
def create_to_edge_transform_and_lower(*args, **kwargs):
kwargs["use_fp16"] = use_fp16
kwargs["online_prepare"] = online_prepare
return ToEdgeTransformAndLower(*args, **kwargs)

# Specialize for Qualcomm
Expand Down
2 changes: 2 additions & 0 deletions backends/test/suite/flow.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -143,11 +143,13 @@ def _load_qnn() -> list[TestFlow]:
QNN_16A4W_TEST_FLOW,
QNN_16A8W_TEST_FLOW,
QNN_8A8W_TEST_FLOW,
QNN_ONLINE_PREPARE_TEST_FLOW,
QNN_TEST_FLOW,
)

return [
QNN_TEST_FLOW,
QNN_ONLINE_PREPARE_TEST_FLOW,
QNN_16A16W_TEST_FLOW,
QNN_16A8W_TEST_FLOW,
QNN_16A4W_TEST_FLOW,
Expand Down
3 changes: 3 additions & 0 deletions backends/test/suite/flows/qualcomm.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
from executorch.backends.qualcomm.quantizer.quantizer import QnnQuantizer, QuantDtype
from executorch.backends.qualcomm.tests.tester import QualcommTester, Quantize
from executorch.backends.test.suite.flow import TestFlow
Expand All@@ -12,12 +12,14 @@
per_channel_linear=False,
is_qat=False,
use_fp16=True,
online_prepare=False,
) -> TestFlow:
if quantize and quant_dtype is None:
raise RuntimeError("Quant dtype must be provided when quantize is true.")

def create_tester(*args, **kwargs) -> QualcommTester:
kwargs["use_fp16"] = (use_fp16,)
kwargs["online_prepare"] = online_prepare
return QualcommTester(*args, **kwargs)

def create_quantize_stage() -> Quantize:
Expand All@@ -41,6 +43,7 @@


QNN_TEST_FLOW = _create_qnn_flow("qnn")
QNN_ONLINE_PREPARE_TEST_FLOW = _create_qnn_flow("qnn_online_prepare", online_prepare=True)
QNN_16A16W_TEST_FLOW = _create_qnn_flow(
"qnn_16a16w", quantize=True, quant_dtype=QuantDtype.use_16a16w, use_fp16=False
)
Expand Down
Loading