Skip to content

Add Neon implementation of std::swap_ranges #5819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Stephan T. Lavavej (StephanTLavavej) merged 7 commits into from
Nov 5, 2025
Merged

Add Neon implementation of std::swap_ranges #5819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7e0a46c
Add Neon implementation of std::swap_ranges
hazzlim Oct 27, 2025
55a3715
Address whitespace & preprocessor review comments
hazzlim Oct 31, 2025
86896d1
Remove unecessary ifdef
hazzlim Nov 3, 2025
250a17a
ARM64 doesn't need legacy `__std_swap_ranges_trivially_swappable`.
StephanTLavavej Nov 4, 2025
a16cb5a
Override /Os for all architectures, before any function defns, cite G…
StephanTLavavej Nov 4, 2025
2239670
Improve perf: Only the 64-byte step needs to loop.
StephanTLavavej Nov 4, 2025
3a42a4f
Reduce the scope of `_Mask_64`.
StephanTLavavej Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion stl/inc/xutility
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ _STL_DISABLE_CLANG_WARNINGS
#define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86
#define _VECTORIZED_SEARCH _VECTORIZED_FOR_X64_X86
#define _VECTORIZED_SEARCH_N _VECTORIZED_FOR_X64_X86
#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86
#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86_ARM64
#define _VECTORIZED_UNIQUE _VECTORIZED_FOR_X64_X86
#define _VECTORIZED_UNIQUE_COPY _VECTORIZED_FOR_X64_X86

Expand Down
105 changes: 95 additions & 10 deletions stl/src/vector_algorithms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,29 @@
#error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp.
#endif

#if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC
#ifndef _DEBUG
#pragma optimize("t", on) // TRANSITION, GH-2108: Override /Os with /Ot for this TU before any function definitions
#endif

#include <__msvc_minmax.hpp>
#include <cstdint>
#include <cstring>
#include <cwchar>
#include <type_traits>

#ifndef _M_ARM64EC
#if !defined(_M_ARM64) && !defined(_M_ARM64EC)
#include <intrin.h>
#include <isa_availability.h>

extern "C" long __isa_enabled;
#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^

#ifndef _DEBUG
#pragma optimize("t", on) // Override /Os with /Ot for this TU
#endif // !defined(_DEBUG)
#endif // ^^^ !defined(_M_ARM64EC) ^^^
#ifdef _M_ARM64
#include <arm64_neon.h>
#endif

namespace {
#ifndef _M_ARM64EC
#if !defined(_M_ARM64) && !defined(_M_ARM64EC)
bool _Use_avx2() noexcept {
return __isa_enabled & (1 << __ISA_AVAILABLE_AVX2);
}
Expand All @@ -51,7 +54,7 @@ namespace {
return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(
reinterpret_cast<const unsigned char*>(_Tail_masks) + (32 - _Count_in_bytes)));
}
#endif // ^^^ !defined(_M_ARM64EC) ^^^
#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^

size_t _Byte_length(const void* const _First, const void* const _Last) noexcept {
return static_cast<const unsigned char*>(_Last) - static_cast<const unsigned char*>(_First);
Expand All @@ -78,6 +81,86 @@ namespace {

extern "C" {

#ifdef _M_ARM64
__declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias(
void* _First1, void* const _Last1, void* _First2) noexcept {
if (_Byte_length(_First1, _Last1) >= 64) {
constexpr size_t _Mask_64 = ~((static_cast<size_t>(1) << 6) - 1);
const void* _Stop_at = _First1;
_Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_64);
do {
const uint8x16_t _Left1 = vld1q_u8(static_cast<uint8_t*>(_First1) + 0);
const uint8x16_t _Left2 = vld1q_u8(static_cast<uint8_t*>(_First1) + 16);
const uint8x16_t _Left3 = vld1q_u8(static_cast<uint8_t*>(_First1) + 32);
const uint8x16_t _Left4 = vld1q_u8(static_cast<uint8_t*>(_First1) + 48);
Comment thread
StephanTLavavej marked this conversation as resolved.
const uint8x16_t _Right1 = vld1q_u8(static_cast<uint8_t*>(_First2) + 0);
const uint8x16_t _Right2 = vld1q_u8(static_cast<uint8_t*>(_First2) + 16);
const uint8x16_t _Right3 = vld1q_u8(static_cast<uint8_t*>(_First2) + 32);
const uint8x16_t _Right4 = vld1q_u8(static_cast<uint8_t*>(_First2) + 48);
vst1q_u8(static_cast<uint8_t*>(_First1) + 0, _Right1);
vst1q_u8(static_cast<uint8_t*>(_First1) + 16, _Right2);
vst1q_u8(static_cast<uint8_t*>(_First1) + 32, _Right3);
vst1q_u8(static_cast<uint8_t*>(_First1) + 48, _Right4);
vst1q_u8(static_cast<uint8_t*>(_First2) + 0, _Left1);
vst1q_u8(static_cast<uint8_t*>(_First2) + 16, _Left2);
vst1q_u8(static_cast<uint8_t*>(_First2) + 32, _Left3);
vst1q_u8(static_cast<uint8_t*>(_First2) + 48, _Left4);
_Advance_bytes(_First1, 64);
_Advance_bytes(_First2, 64);
} while (_First1 != _Stop_at);
}

if (_Byte_length(_First1, _Last1) >= 32) {
const uint8x16_t _Left1 = vld1q_u8(static_cast<uint8_t*>(_First1) + 0);
const uint8x16_t _Left2 = vld1q_u8(static_cast<uint8_t*>(_First1) + 16);
const uint8x16_t _Right1 = vld1q_u8(static_cast<uint8_t*>(_First2) + 0);
const uint8x16_t _Right2 = vld1q_u8(static_cast<uint8_t*>(_First2) + 16);
vst1q_u8(static_cast<uint8_t*>(_First1) + 0, _Right1);
vst1q_u8(static_cast<uint8_t*>(_First1) + 16, _Right2);
vst1q_u8(static_cast<uint8_t*>(_First2) + 0, _Left1);
vst1q_u8(static_cast<uint8_t*>(_First2) + 16, _Left2);
_Advance_bytes(_First1, 32);
_Advance_bytes(_First2, 32);
}

if (_Byte_length(_First1, _Last1) >= 16) {
const uint8x16_t _Left = vld1q_u8(static_cast<uint8_t*>(_First1));
const uint8x16_t _Right = vld1q_u8(static_cast<uint8_t*>(_First2));
vst1q_u8(static_cast<uint8_t*>(_First1), _Right);
vst1q_u8(static_cast<uint8_t*>(_First2), _Left);
_Advance_bytes(_First1, 16);
_Advance_bytes(_First2, 16);
}

if (_Byte_length(_First1, _Last1) >= 8) {
const uint8x8_t _Left = vld1_u8(static_cast<uint8_t*>(_First1));
const uint8x8_t _Right = vld1_u8(static_cast<uint8_t*>(_First2));
vst1_u8(static_cast<uint8_t*>(_First1), _Right);
vst1_u8(static_cast<uint8_t*>(_First2), _Left);
_Advance_bytes(_First1, 8);
_Advance_bytes(_First2, 8);
}

if (_Byte_length(_First1, _Last1) >= 4) {
uint32x2_t _Left = vdup_n_u32(0);
uint32x2_t _Right = vdup_n_u32(0);
_Left = vld1_lane_u32(static_cast<uint32_t*>(_First1), _Left, 0);
_Right = vld1_lane_u32(static_cast<uint32_t*>(_First2), _Right, 0);
vst1_lane_u32(static_cast<uint32_t*>(_First1), _Right, 0);
vst1_lane_u32(static_cast<uint32_t*>(_First2), _Left, 0);
_Advance_bytes(_First1, 4);
_Advance_bytes(_First2, 4);
}

auto _First1c = static_cast<unsigned char*>(_First1);
auto _First2c = static_cast<unsigned char*>(_First2);
for (; _First1c != _Last1; ++_First1c, ++_First2c) {
const unsigned char _Ch = *_First1c;
*_First1c = *_First2c;
*_First2c = _Ch;
}
}
#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv
__declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias(
void* _First1, void* const _Last1, void* _First2) noexcept {
#ifndef _M_ARM64EC
Expand Down Expand Up @@ -157,15 +240,17 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias(
}
}

// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility
// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility (x64/x86/ARM64EC)
void* __cdecl __std_swap_ranges_trivially_swappable(
void* const _First1, void* const _Last1, void* const _First2) noexcept {
__std_swap_ranges_trivially_swappable_noalias(_First1, _Last1, _First2);
return static_cast<char*>(_First2) + (static_cast<char*>(_Last1) - static_cast<char*>(_First1));
}
#endif // ^^^ !defined(_M_ARM64) ^^^

} // extern "C"

#ifndef _M_ARM64
namespace {
namespace _Rotating {
void _Swap_3_ranges(void* _First1, void* const _Last1, void* _First2, void* _First3) noexcept {
Expand Down Expand Up @@ -7694,4 +7779,4 @@ __declspec(noalias) bool __stdcall __std_bitset_from_string_2(void* const _Dest,
}

} // extern "C"
#endif // defined(_M_IX86) || defined(_M_X64)
#endif // ^^^ !defined(_M_ARM64) ^^^