[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
facebook-github-bot merged 9 commits into from
Mar 14, 2025
Merged

[XNNPACK][Weights Cache] Enable in XNNPACK #9155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
09c7427
[XNNPACK][Weights Cache] Enable in XNNPACK
mcr229 Mar 11, 2025
aa8e0d1
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 11, 2025
7153f19
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 11, 2025
bde775e
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 12, 2025
f3a20db
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 13, 2025
34f3c6f
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 14, 2025
a26a962
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 14, 2025
001657b
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 14, 2025
c4c62de
Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"
mcr229 Mar 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions backends/xnnpack/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
"Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down
72 changes: 60 additions & 12 deletions backends/xnnpack/runtime/XNNCompiler.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
GraphPtr flatbuffer_graph,
const uint8_t* constant_data_ptr,
const NamedDataMap* named_data_map,
std::vector<FreeableBuffer>& loaded_buffers_from_map) {
std::vector<FreeableBuffer>& freeable_buffers,
XNNWeightsCache* weights_cache) {
auto buffer_idx = tensor_value->constant_buffer_idx();
if (buffer_idx) {
if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
return constant_data_ptr + offset;
} else {
const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
Result<const uint8_t*> data_ptr =
weights_cache->load_unpacked_data(data_name);
if (!data_ptr.ok()) {
ET_LOG(Error, "Failed to load weights from cache");
return nullptr;
}
return data_ptr.get();
#else
Result<FreeableBuffer> buffer =
named_data_map->get_data(data_name.c_str());
if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
}
const uint8_t* data_ptr =
static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
return data_ptr;
#endif
}
}
}
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
std::vector<uint32_t>& output_ids,
CompileAllocator& allocator,
const NamedDataMap* named_data_map,
std::vector<FreeableBuffer>& loaded_buffers_from_map) {
std::vector<FreeableBuffer>& freeable_buffers,
XNNWeightsCache* weights_cache) {
const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
flatbuffer_graph,
constant_data_ptr,
named_data_map,
loaded_buffers_from_map);
freeable_buffers,
weights_cache);

xnn_status status;
// The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
const void* buffer_pointer,
size_t num_bytes,
XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
const uint8_t* flatbuffer_data = nullptr;
const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
// Invalid ids do not need to be remapped
remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

// If weight cache is not on we hold onto all the unpacked buffers
// and we free them at the end
std::vector<FreeableBuffer> unpacked_buffers;

// External Ids for inputs and outputs
std::vector<uint32_t> input_ids;
std::vector<uint32_t> output_ids;
Error err = Error::Ok;
std::vector<FreeableBuffer> loaded_buffers_from_map;
for (auto value : *flatbuffer_graph->xvalues()) {
err = defineTensor(
subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
output_ids,
compile_allocator,
named_data_map,
loaded_buffers_from_map);
unpacked_buffers,
weights_cache);

if (err != Error::Ok) {
return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

xnn_runtime_t runtime_ptr = nullptr;

// XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
// just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
ET_CHECK_OR_RETURN_ERROR(
unpacked_buffers.size() == 0,
Internal,
"Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
xnn_weights_cache_t weights_cache_ptr =
weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
: nullptr;
#else
xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
ET_CHECK_OR_RETURN_ERROR(
workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
status = xnn_create_runtime_v4(
subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
workspace,
::executorch::extension::threadpool::get_pthreadpool(),
runtime_flags,
&runtime_ptr);
#else
status = xnn_create_runtime_v3(
subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
::executorch::extension::threadpool::get_pthreadpool(),
runtime_flags,
&runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
"XNN Runtime creation failed with code: %s",
xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
auto packed_weights_names = weights_cache->finalize_for_runtime();
ET_CHECK_OR_RETURN_ERROR(
packed_weights_names.ok(),
Internal,
"Failed to finalize weights cache after creating the xnn runtime")
#else
for (auto& buffer : unpacked_buffers) {
buffer.Free();
}
Result<std::vector<std::string>> packed_weights_names =
std::vector<std::string>();
#endif

err = executor->initialize( // NOLINT: runtime_ptr is non-null
runtime_ptr,
std::move(input_ids),
std::move(output_ids));
std::move(output_ids),
std::move(packed_weights_names.get()));

return err;
};
Expand Down
10 changes: 4 additions & 6 deletions backends/xnnpack/runtime/XNNCompiler.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
const void* buffer_pointer,
size_t num_bytes,
XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down
4 changes: 3 additions & 1 deletion backends/xnnpack/runtime/XNNExecutor.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
xnn_runtime_t runtime,
std::vector<uint32_t>&& input_ids,
std::vector<uint32_t>&& output_ids) {
std::vector<uint32_t>&& output_ids,
std::vector<std::string>&& packed_data_names) {
runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
std::sort(output_ids_.begin(), output_ids_.end());

externals_.resize(input_ids_.size() + output_ids_.size());
packed_data_names_ = std::move(packed_data_names);

return Error::Ok;
}
Expand Down
8 changes: 7 additions & 1 deletion backends/xnnpack/runtime/XNNExecutor.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
std::vector<uint32_t> input_ids_;
std::vector<uint32_t> output_ids_;
std::vector<xnn_external_value> externals_;
std::vector<std::string> packed_data_names_;

public:
XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
return output_ids_.size();
}

inline std::vector<std::string> get_packed_data_names() {
return packed_data_names_;
}

/**
* Initialize the XNNExecutor with a given runtime and input/output ids.
* The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
ET_NODISCARD executorch::runtime::Error initialize(
xnn_runtime_t runtime,
std::vector<uint32_t>&& input_ids,
std::vector<uint32_t>&& output_ids);
std::vector<uint32_t>&& output_ids,
std::vector<std::string>&& packed_data_names);

/**
* Prepares the arguments for runtime graph execution.
Expand Down
42 changes: 35 additions & 7 deletions backends/xnnpack/runtime/XNNPACKBackend.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
*/

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
}

const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
// This is needed to serialize access to xnn_create_runtime which is not
// thread safe. This can heppen when multiple threads call init() on
// the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
weights_cache_->initialize_for_runtime(
context.get_runtime_allocator(), named_data_map);
#endif

// Executor has been allocated but not constructed, ensure that runtime_ is
// nullptr by constructing it in place here. NOTE: Since we use placement
// new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
processed->data(),
processed->size(),
executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
// This backend does not need its processed data after compiling the model.
processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

// Prepare Inputs/Outputs and Propagate Input Shapes
Error err = executor->prepare_args(args);
if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

void destroy(DelegateHandle* handle) const override {
if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
// This is needed to serialize access to xnn_delete_runtime which is not
// thread safe. This can heppen when multiple threads call destroy() on
// the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
const std::lock_guard<std::mutex> lock_weights_cache(
weights_cache_mutex_);
weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
// XNNExecutor is not trivially destructible. Since this was constructed
// manually in init(), we must destroy it manually here.
executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
nullptr,
&xnn_release_workspace};

// Weights cache is global to all delegate instances.
mutable std::mutex weights_cache_mutex_;
std::unique_ptr<XNNWeightsCache> weights_cache_ =
std::make_unique<XNNWeightsCache>();

// Lock Hiearchy for Mutexes:
// workspace_mutex_
// weights_cache_mutex_
};

namespace {
Expand Down
10 changes: 7 additions & 3 deletions backends/xnnpack/targets.bzl
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
Disable if someone explictly specified a config option,
else Enable otherwise
"""
if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
return []
preprocessor_flags = []
if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

# Enable if not disabled through config
return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
return preprocessor_flags

def define_common_targets():
runtime.cxx_library(
Expand Down
3 changes: 2 additions & 1 deletion backends/xnnpack/test/runtime/test_xnnexecutor.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
},
{
1,
}),
},
{}),
Error::Ok);
TensorFactory<executorch::aten::ScalarType::Int> tf;
auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
[XNNPACK][Weights Cache] Enable in XNNPACK#9155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

facebook-github-bot
merged 9 commits into

from

Mar 14, 2025
 Merged


[XNNPACK][Weights Cache] Enable in XNNPACK
#9155
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
09c7427

[XNNPACK][Weights Cache] Enable in XNNPACK


mcr229 Mar 11, 2025
aa8e0d1

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
7153f19

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 11, 2025
bde775e

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 12, 2025
f3a20db

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 13, 2025
34f3c6f

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
a26a962

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
001657b

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025
c4c62de

Update on "[XNNPACK][Weights Cache] Enable in XNNPACK"


mcr229 Mar 14, 2025

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


13 changes: 13 additions & 0 deletions
backends/xnnpack/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,19 @@ option(EXECUTORCH_XNNPACK_SHARED_WORKSPACE
# Keeping this OFF by default due to regressions in decode and model load with
# kleidi kernels
option(EXECUTORCH_XNNPACK_ENABLE_KLEIDI "Enable Arm Kleidi kernels" OFF)

# Turning this on cache weights between partitions and methods. If weights
# are shared across methods/partitions then this can reduce load time and
# memory usage

# Keeping this off maintains existing behavior. Turning this on serializes
# execution and initialization of delegates, to be revisited
option(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE
 "Enable weights cache to cache and manage all packed weights" OFF)

if(EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE)
 add_definitions(-DENABLE_XNNPACK_WEIGHTS_CACHE)
endif()
if(EXECUTORCH_XNNPACK_SHARED_WORKSPACE)
 add_definitions(-DENABLE_XNNPACK_SHARED_WORKSPACE)
endif()
Expand Down

72 changes: 60 additions & 12 deletions
backends/xnnpack/runtime/XNNCompiler.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -11,7 +11,9 @@
#include <executorch/backends/xnnpack/serialization/schema_generated.h>
#include <executorch/extension/threadpool/threadpool.h>
#include <executorch/runtime/executor/pte_data_map.h>
#include <string>
#include <unordered_map>
#include <vector>

#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wglobal-constructors"
Expand DownExpand Up@@ -167,7 +169,8 @@ const uint8_t* getConstantDataPtr(
 GraphPtr flatbuffer_graph,
 const uint8_t* constant_data_ptr,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 auto buffer_idx = tensor_value->constant_buffer_idx();
 if (buffer_idx) {
 if (!constant_data_ptr) {
Expand All@@ -187,6 +190,15 @@ const uint8_t* getConstantDataPtr(
 return constant_data_ptr + offset;
 } else {
 const std::string& data_name = constant_data_offset->named_key()->str();
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 Result<const uint8_t*> data_ptr =
 weights_cache->load_unpacked_data(data_name);
 if (!data_ptr.ok()) {
 ET_LOG(Error, "Failed to load weights from cache");
 return nullptr;
 }
 return data_ptr.get();
#else
 Result<FreeableBuffer> buffer =
 named_data_map->get_data(data_name.c_str());
 if (!buffer.ok()) {
Expand All@@ -198,8 +210,9 @@ const uint8_t* getConstantDataPtr(
 }
 const uint8_t* data_ptr =
 static_cast<const uint8_t*>(buffer.get().data());
loaded_buffers_from_map.push_back(std::move(buffer.get()));
freeable_buffers.push_back(std::move(buffer.get()));
 return data_ptr;
#endif
 }
 }
 }
Expand All@@ -222,7 +235,8 @@ Error defineTensor(
 std::vector<uint32_t>& output_ids,
 CompileAllocator& allocator,
 const NamedDataMap* named_data_map,
 std::vector<FreeableBuffer>& loaded_buffers_from_map) {
 std::vector<FreeableBuffer>& freeable_buffers,
 XNNWeightsCache* weights_cache) {
 const fb_xnnpack::XNNTensorValue* tensor_value = nullptr;
 const fb_xnnpack::XNNQuantizedTensorValue* qtensor_value = nullptr;

Expand DownExpand Up@@ -264,7 +278,8 @@ Error defineTensor(
 flatbuffer_graph,
 constant_data_ptr,
 named_data_map,
 loaded_buffers_from_map);
 freeable_buffers,
 weights_cache);

 xnn_status status;
 // The type we might have to convert to
Expand DownExpand Up@@ -1999,9 +2014,9 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
MemoryAllocator* runtime_allocator,
const NamedDataMap* named_data_map,
xnn_workspace_t workspace) {
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map) {
 Result<XNNHeader> header = XNNHeader::Parse(buffer_pointer, num_bytes);
 const uint8_t* flatbuffer_data = nullptr;
 const uint8_t* constant_data = nullptr;
Expand DownExpand Up@@ -2065,11 +2080,14 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 // Invalid ids do not need to be remapped
 remapped_ids.emplace(XNN_INVALID_VALUE_ID, XNN_INVALID_VALUE_ID);

 // If weight cache is not on we hold onto all the unpacked buffers
 // and we free them at the end
 std::vector<FreeableBuffer> unpacked_buffers;

 // External Ids for inputs and outputs
 std::vector<uint32_t> input_ids;
 std::vector<uint32_t> output_ids;
 Error err = Error::Ok;
 std::vector<FreeableBuffer> loaded_buffers_from_map;
 for (auto value : *flatbuffer_graph->xvalues()) {
 err = defineTensor(
 subgraph.get(),
Expand All@@ -2081,7 +2099,8 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 output_ids,
 compile_allocator,
 named_data_map,
 loaded_buffers_from_map);
 unpacked_buffers,
 weights_cache);

 if (err != Error::Ok) {
 return err;
Expand All@@ -2103,20 +2122,34 @@ ET_NODISCARD Error XNNCompiler::compileModel(

 xnn_runtime_t runtime_ptr = nullptr;

 // XNNWeightsCache if weights cache is not enabled, then XNNWeightsCache
 // just manages the unpacked weights until the runtime is created.
#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 ET_CHECK_OR_RETURN_ERROR(
 unpacked_buffers.size() == 0,
 Internal,
 "Weight Cache is enabled, which means unpacked buffers should be owned by the cache");
 xnn_weights_cache_t weights_cache_ptr =
 weights_cache->get_num_unpacked_data() > 0 ? weights_cache->get()
 : nullptr;
#else
 xnn_weights_cache_t weights_cache_ptr = nullptr;
#endif

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 ET_CHECK_OR_RETURN_ERROR(
 workspace != nullptr, Internal, "Failed to initialize XNNPACK workspace");
 status = xnn_create_runtime_v4(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 workspace,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
#else
 status = xnn_create_runtime_v3(
 subgraph.get(),
/*weight_cache=*/nullptr, // TODO - support weight cache
weights_cache_ptr,
 ::executorch::extension::threadpool::get_pthreadpool(),
 runtime_flags,
 &runtime_ptr);
Expand All@@ -2128,10 +2161,25 @@ ET_NODISCARD Error XNNCompiler::compileModel(
 "XNN Runtime creation failed with code: %s",
 xnn_status_to_string(status));

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 auto packed_weights_names = weights_cache->finalize_for_runtime();
 ET_CHECK_OR_RETURN_ERROR(
 packed_weights_names.ok(),
 Internal,
 "Failed to finalize weights cache after creating the xnn runtime")
#else
 for (auto& buffer : unpacked_buffers) {
 buffer.Free();
 }
 Result<std::vector<std::string>> packed_weights_names =
 std::vector<std::string>();
#endif

 err = executor->initialize( // NOLINT: runtime_ptr is non-null
 runtime_ptr,
 std::move(input_ids),
 std::move(output_ids));
 std::move(output_ids),
 std::move(packed_weights_names.get()));

 return err;
};
Expand Down

10 changes: 4 additions & 6 deletions
backends/xnnpack/runtime/XNNCompiler.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -9,11 +9,9 @@
#pragma once

#include <executorch/backends/xnnpack/runtime/XNNExecutor.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/platform/compiler.h>

#include <xnnpack.h>
#include <memory>
#include <vector>

namespace executorch {
namespace backends {
Expand All@@ -29,9 +27,9 @@ class XNNCompiler {
 const void* buffer_pointer,
 size_t num_bytes,
 XNNExecutor* executor,
executorch::runtime::MemoryAllocator* runtime_allocator,
const executorch::runtime::NamedDataMap* named_data_map,
xnn_workspace_t workspace);
XNNWeightsCache* weights_cache,
xnn_workspace_t workspace,
const NamedDataMap* named_data_map);
};

} // namespace delegate
Expand Down

4 changes: 3 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -30,7 +30,8 @@ using executorch::runtime::kTensorDimensionLimit;
ET_NODISCARD Error XNNExecutor::initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids) {
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names) {
 runtime_ = std::unique_ptr<xnn_runtime, decltype(&xnn_delete_runtime)>(
 runtime, xnn_delete_runtime);

Expand All@@ -51,6 +52,7 @@ ET_NODISCARD Error XNNExecutor::initialize(
 std::sort(output_ids_.begin(), output_ids_.end());

 externals_.resize(input_ids_.size() + output_ids_.size());
 packed_data_names_ = std::move(packed_data_names);

 return Error::Ok;
}
Expand Down

8 changes: 7 additions & 1 deletion
backends/xnnpack/runtime/XNNExecutor.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ class XNNExecutor {
 std::vector<uint32_t> input_ids_;
 std::vector<uint32_t> output_ids_;
 std::vector<xnn_external_value> externals_;
 std::vector<std::string> packed_data_names_;

 public:
 XNNExecutor() = default;
Expand All@@ -46,6 +47,10 @@ class XNNExecutor {
 return output_ids_.size();
 }

 inline std::vector<std::string> get_packed_data_names() {
 return packed_data_names_;
 }

 /**
 * Initialize the XNNExecutor with a given runtime and input/output ids.
 * The input/output ids are expected to be sorted in order of their
Expand All@@ -54,7 +59,8 @@ class XNNExecutor {
 ET_NODISCARD executorch::runtime::Error initialize(
 xnn_runtime_t runtime,
 std::vector<uint32_t>&& input_ids,
 std::vector<uint32_t>&& output_ids);
 std::vector<uint32_t>&& output_ids,
 std::vector<std::string>&& packed_data_names);

 /**
 * Prepares the arguments for runtime graph execution.
Expand Down

42 changes: 35 additions & 7 deletions
backends/xnnpack/runtime/XNNPACKBackend.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -7,6 +7,7 @@
 */

#include <executorch/backends/xnnpack/runtime/XNNCompiler.h>
#include <executorch/backends/xnnpack/runtime/XNNWeightsCache.h>
#include <executorch/runtime/backend/interface.h>
#include <executorch/runtime/core/error.h>
#include <executorch/runtime/core/evalue.h>
Expand All@@ -20,6 +21,7 @@
namespace executorch {
namespace backends {

using executorch::backends::xnnpack::delegate::XNNWeightsCache;
using executorch::runtime::ArrayRef;
using executorch::runtime::Backend;
using executorch::runtime::BackendExecutionContext;
Expand DownExpand Up@@ -81,13 +83,18 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 }

 const NamedDataMap* named_data_map = context.get_named_data_map();

#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_create_runtime which is not
 // thread safe. This can heppen when multiple threads call init() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weight_cache(weights_cache_mutex_);
 weights_cache_->initialize_for_runtime(
 context.get_runtime_allocator(), named_data_map);
#endif

 // Executor has been allocated but not constructed, ensure that runtime_ is
 // nullptr by constructing it in place here. NOTE: Since we use placement
 // new and since this type is not trivially destructible, we must call the
Expand All@@ -97,9 +104,9 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 processed->data(),
 processed->size(),
 executor,
context.get_runtime_allocator(),
named_data_map,
workspace_.get());
weights_cache_.get(),
workspace_.get(),
named_data_map);
 // This backend does not need its processed data after compiling the model.
 processed->Free();

Expand All@@ -125,6 +132,10 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(weights_cache_mutex_);
#endif

 // Prepare Inputs/Outputs and Propagate Input Shapes
 Error err = executor->prepare_args(args);
 if (err != Error::Ok) {
Expand All@@ -145,16 +156,24 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {

 void destroy(DelegateHandle* handle) const override {
 if (handle != nullptr) {
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 // This is needed to serialize access to xnn_delete_runtime which is not
 // thread safe. This can heppen when multiple threads call destroy() on
 // the same backend instance.
#ifdef ENABLE_XNNPACK_SHARED_WORKSPACE
 const std::lock_guard<std::mutex> lock(workspace_mutex_);
#endif

 auto executor = static_cast<xnnpack::delegate::XNNExecutor*>(handle);

#ifdef ENABLE_XNNPACK_PROFILING
 executor->print_avg_op_timings();
#endif

#ifdef ENABLE_XNNPACK_WEIGHTS_CACHE
 const std::lock_guard<std::mutex> lock_weights_cache(
 weights_cache_mutex_);
 weights_cache_->delete_packed_data(executor->get_packed_data_names());
#endif
 // XNNExecutor is not trivially destructible. Since this was constructed
 // manually in init(), we must destroy it manually here.
 executor->~XNNExecutor();
Expand All@@ -167,6 +186,15 @@ class XnnpackBackend final : public ::executorch::runtime::BackendInterface {
 std::unique_ptr<xnn_workspace, decltype(&xnn_release_workspace)> workspace_{
 nullptr,
 &xnn_release_workspace};

 // Weights cache is global to all delegate instances.
 mutable std::mutex weights_cache_mutex_;
 std::unique_ptr<XNNWeightsCache> weights_cache_ =
 std::make_unique<XNNWeightsCache>();

 // Lock Hiearchy for Mutexes:
 // workspace_mutex_
 // weights_cache_mutex_
};

namespace {
Expand Down

10 changes: 7 additions & 3 deletions
backends/xnnpack/targets.bzl

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,11 +6,15 @@ def _get_preprocessor_flags():
 Disable if someone explictly specified a config option,
 else Enable otherwise
 """
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") == "0":
 return []
 preprocessor_flags = []
 if native.read_config("executorch", "xnnpack_workspace_sharing", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_SHARED_WORKSPACE")

 if native.read_config("executorch", "xnnpack_weights_cache", "0") != "0":
 preprocessor_flags.append("-DENABLE_XNNPACK_WEIGHTS_CACHE")

 # Enable if not disabled through config
 return ["-DENABLE_XNNPACK_SHARED_WORKSPACE"]
 return preprocessor_flags

def define_common_targets():
 runtime.cxx_library(
Expand Down

3 changes: 2 additions & 1 deletion
backends/xnnpack/test/runtime/test_xnnexecutor.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -74,7 +74,8 @@ TEST(XNNExecutorTest, ArgumentWithTooManyDimensions) {
 },
 {
 1,
 }),
 },
 {}),
 Error::Ok);
 TensorFactory<executorch::aten::ScalarType::Int> tf;
 auto input_tensor = tf.make({1, 1, 1, 1, 1, 1, 1, 1, 1}, {42});
Expand Down