Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 1.1k
Cortex-M: Use q/dq ops in Arm Ethos Runner#10782
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Uh oh!
There was an error while loading. Please reload this page.
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
2c25378
Arm backend: Add quantize_per_tensor MVE impl
digantdesai 985a077
Arm backend: Add dquantize_per_tensor MVE version
digantdesai 5d92fc2
CMake: Add new Cortex_M top level build option
digantdesai 428d836
Arm backend: Build Cortex-M backend with ET
digantdesai cc7bfe3
Arm backend: Add cortex_m backend dep to the runner
digantdesai 7c0c99b
Arm backend: Use quant model for bundled PTE when available
digantdesai 376aab2
Arm backend: Use Cortex-M backend
digantdesai d75b89f
Arm backends: New quantized FVP tests for cortex-M
digantdesai File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # Copyright (c) Meta Platforms, Inc. and affiliates. | ||
| # All rights reserved. | ||
| # | ||
| # This source code is licensed under the BSD-style license found in the | ||
| # LICENSE file in the root directory of this source tree. | ||
| # Kernel library for Cortex-M operators. Please keep this file formatted by running: | ||
| # ~~~ | ||
| # cmake-format -i CMakeLists.txt | ||
| # ~~~ | ||
| cmake_minimum_required(VERSION 3.19) | ||
| set(CMAKE_EXPORT_COMPILE_COMMANDS ON) | ||
| if(NOT CMAKE_CXX_STANDARD) | ||
| set(CMAKE_CXX_STANDARD 17) | ||
| endif() | ||
| # Source root directory for executorch. | ||
| if(NOT EXECUTORCH_ROOT) | ||
| set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) | ||
| endif() | ||
| include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) | ||
| include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) | ||
| if(NOT PYTHON_EXECUTABLE) | ||
| resolve_python_executable() | ||
| endif() | ||
| # Cortex-M ops kernel sources | ||
| set(_cortex_m_kernels__srcs | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/ops/op_quantize_per_tensor.cpp | ||
| ${CMAKE_CURRENT_SOURCE_DIR}/ops/op_dequantize_per_tensor.cpp | ||
| ) | ||
| # Generate C++ bindings to register kernels into Executorch (for runtime). | ||
| # Here select all ops in operators.yaml | ||
| set(_yaml_file ${CMAKE_CURRENT_LIST_DIR}/ops/operators.yaml) | ||
| gen_selected_ops(LIB_NAME "cortex_m_ops_lib" OPS_SCHEMA_YAML "${_yaml_file}") | ||
| # Generate bindings for the kernels | ||
| generate_bindings_for_kernels( | ||
| LIB_NAME "cortex_m_ops_lib" CUSTOM_OPS_YAML "${_yaml_file}" | ||
| ) | ||
| message("Generated files ${gen_command_sources}") | ||
| # Build a library for _cortex_m_kernels_srcs | ||
| add_library(cortex_m_kernels ${_cortex_m_kernels__srcs}) | ||
| target_link_libraries(cortex_m_kernels PRIVATE executorch) | ||
| target_compile_options(cortex_m_kernels PUBLIC ${_common_compile_options}) | ||
| # cortex_m_ops_lib: Register Cortex-M ops kernels into Executorch runtime | ||
| gen_operators_lib( | ||
| LIB_NAME "cortex_m_ops_lib" KERNEL_LIBS cortex_m_kernels DEPS executorch | ||
| ) | ||
| install( | ||
| TARGETS cortex_m_kernels cortex_m_ops_lib | ||
| DESTINATION lib | ||
| PUBLIC_HEADER DESTINATION include/executorch/backends/cortex_m/ops/ | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -41,13 +41,13 @@ void check_quantize_args( | ||
| "input.scalar_type() %" PRId8 " is not float type", | ||
| static_cast<int8_t>(input.scalar_type())); | ||
| // Check output dtype is int8 (Char) | ||
| // Check output dtype is int8 | ||
| ET_CHECK_MSG( | ||
| out.scalar_type() == ScalarType::Char, | ||
| "out.scalar_type() %" PRId8 " is not int8 (Char)", | ||
| static_cast<int8_t>(out.scalar_type())); | ||
| // Check dtype is int8 (Char) | ||
| // Check dtype is int8 | ||
| ET_CHECK_MSG( | ||
| dtype == ScalarType::Char, | ||
| "dtype %" PRId8 " is not int8 (Char)", | ||
| @@ -75,18 +75,18 @@ void check_quantize_args( | ||
| /** | ||
| * Scalar implementation of quantization for a single value. | ||
| */ | ||
| template <typename T, typename K> | ||
| T quantize_val( | ||
| float inv_scale, | ||
| template <typename Q, typename F> | ||
AdrianLundell marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| Q quantize_val( | ||
| F inv_scale, | ||
| int32_t zero_point, | ||
| K value, | ||
| F value, | ||
| int64_t quant_min, | ||
| int64_t quant_max) { | ||
| int32_t qvalue = | ||
| zero_point + static_cast<int32_t>(std::nearbyint(inv_scale * value)); | ||
| qvalue = std::max<int32_t>(qvalue, static_cast<int32_t>(quant_min)); | ||
| qvalue = std::min<int32_t>(qvalue, static_cast<int32_t>(quant_max)); | ||
| return static_cast<T>(qvalue); | ||
| return static_cast<Q>(qvalue); | ||
| } | ||
| } // namespace | ||
| @@ -123,16 +123,97 @@ Tensor& quantize_per_tensor_out( | ||
| int8_t* out_data = out.mutable_data_ptr<int8_t>(); | ||
| const size_t numel = input.numel(); | ||
| size_t i = 0; | ||
| #if defined(HAS_HELIUM_SIMD) | ||
| // Helium MVE implementation for float32 to int8 quantization | ||
| #Error "Implement MVE version!" | ||
| #else | ||
| // Scalar implementation for float32 to int8 quantization | ||
| for (size_t i = 0; i < numel; i++) { | ||
| out_data[i] = | ||
| quantize_val<int8_t, float>(inv_scale, zp, input_data[i], qmin, qmax); | ||
| // Helium MVE implementation for float32 to int8 quantization | ||
| static uint8x16_t voffset{ | ||
| 0x0, | ||
| 0x8, | ||
| 0x4, | ||
| 0xC, | ||
| 0x1, | ||
| 0x9, | ||
| 0x5, | ||
| 0xD, | ||
| 0x2, | ||
| 0xA, | ||
| 0x6, | ||
| 0xE, | ||
| 0x3, | ||
| 0xB, | ||
| 0x7, | ||
| 0xF}; | ||
| float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); | ||
| // Magic number for float to int conversion, round to nearest even integer | ||
AdrianLundell marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| // int magic_round(float f): interpret_as_int32(f + magic_float) - magic_int | ||
| // where, | ||
| // magic_float = 12582912.0f = (2 ** 23 + 2 ** 22) = (1.5 * 2 ** 23) | ||
| // magic_int = 1262485504 = 0x4B400000 = bit_pattern_as_int32(magic_float) | ||
| float magic_float = 12582912.0f; | ||
| int32_t magic_int = 1262485504; | ||
| float32x4_t vmagic_float = vdupq_n_f32(magic_float); | ||
| int32x4_t vmagic_int_less_zp = | ||
| vdupq_n_s32(magic_int - static_cast<int32_t>(zp)); | ||
| int16x8_t vqmin = vdupq_n_s16(qmin); | ||
| int16x8_t vqmax = vdupq_n_s16(qmax); | ||
| // TODO: Measure performnce, we are spilling | ||
| for (; i + 15 < numel; i += 16) { | ||
| float32x4_t in_0123 = vldrwq_f32(input_data + 0); | ||
| float32x4_t in_4567 = vldrwq_f32(input_data + 4); | ||
| float32x4_t in_89AB = vldrwq_f32(input_data + 8); | ||
| float32x4_t in_CDEF = vldrwq_f32(input_data + 12); | ||
| float32x4_t outf_0123 = vfmaq_f32(vmagic_float, in_0123, inv_scale_vec); | ||
| float32x4_t outf_4567 = vfmaq_f32(vmagic_float, in_4567, inv_scale_vec); | ||
| float32x4_t outf_89AB = vfmaq_f32(vmagic_float, in_89AB, inv_scale_vec); | ||
| float32x4_t outf_CDEF = vfmaq_f32(vmagic_float, in_CDEF, inv_scale_vec); | ||
| int32x4_t out_0123 = | ||
| vsubq_s32(vreinterpretq_s32_f32(outf_0123), vmagic_int_less_zp); | ||
| int32x4_t out_4567 = | ||
| vsubq_s32(vreinterpretq_s32_f32(outf_4567), vmagic_int_less_zp); | ||
| int32x4_t out_89AB = | ||
| vsubq_s32(vreinterpretq_s32_f32(outf_89AB), vmagic_int_less_zp); | ||
| int32x4_t out_CDEF = | ||
| vsubq_s32(vreinterpretq_s32_f32(outf_CDEF), vmagic_int_less_zp); | ||
| int16x8_t out_04152637; | ||
| int16x8_t out_8C9DAEBF; | ||
| out_04152637 = vmovnbq_s32(out_04152637, out_0123); | ||
| out_04152637 = vmovntq_s32(out_04152637, out_4567); | ||
| out_8C9DAEBF = vmovnbq_s32(out_8C9DAEBF, out_89AB); | ||
| out_8C9DAEBF = vmovntq_s32(out_8C9DAEBF, out_CDEF); | ||
| int16x8_t out_04152637_clamped = | ||
| vminq_s16(vmaxq_s16(out_04152637, vqmin), vqmax); | ||
| int16x8_t out_8C9DAEBF_clamped = | ||
| vminq_s16(vmaxq_s16(out_8C9DAEBF, vqmin), vqmax); | ||
| int8x16_t out_084C195D2A6E3B7F; | ||
| out_084C195D2A6E3B7F = | ||
| vmovnbq_s16(out_084C195D2A6E3B7F, out_04152637_clamped); | ||
| out_084C195D2A6E3B7F = | ||
| vmovntq_s16(out_084C195D2A6E3B7F, out_8C9DAEBF_clamped); | ||
| vstrbq_scatter_offset_s8(out_data, voffset, out_084C195D2A6E3B7F); | ||
| input_data += 16; | ||
| out_data += 16; | ||
| } | ||
| #endif // defined(HAS_HELIUM_SIMD) | ||
| for (; i < numel; i++) { | ||
| *out_data = | ||
| quantize_val<int8_t, float>(inv_scale, zp, *input_data, qmin, qmax); | ||
| input_data++; | ||
| out_data++; | ||
| } | ||
| #endif | ||
| return out; | ||
| } | ||
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.