From 7e0a46cb544ae704a893e03248543a2b695d80f3 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Mon, 27 Oct 2025 23:56:51 +0000 Subject: [PATCH 1/7] Add Neon implementation of std::swap_ranges Add an implementation of std::swap_ranges using Neon intrinsics. --- stl/inc/xutility | 2 +- stl/src/vector_algorithms.cpp | 120 ++++++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index da3e663bdec..17feb4cd9c5 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -100,7 +100,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_SEARCH _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_SEARCH_N _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_UNIQUE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_UNIQUE_COPY _VECTORIZED_FOR_X64_X86 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ea38e0e39d7..8f570215c55 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5,14 +5,14 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC +#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) // NB: includes _M_ARM64EC #include <__msvc_minmax.hpp> #include #include #include #include -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) #include #include @@ -21,10 +21,14 @@ extern "C" long __isa_enabled; #ifndef _DEBUG #pragma optimize("t", on) // Override /Os with /Ot for this TU #endif // !defined(_DEBUG) -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ + +#ifdef _M_ARM64 +#include +#endif namespace { -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) bool _Use_avx2() noexcept { return __isa_enabled & (1 << __ISA_AVAILABLE_AVX2); } @@ -51,7 +55,7 @@ namespace { return _mm256_loadu_si256(reinterpret_cast( reinterpret_cast(_Tail_masks) + (32 - _Count_in_bytes))); } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ size_t _Byte_length(const void* const _First, const void* const _Last) noexcept { return static_cast(_Last) - static_cast(_First); @@ -78,6 +82,107 @@ namespace { extern "C" { +#ifdef _M_ARM64 +__declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( + void* _First1, void* const _Last1, void* _First2) noexcept { + + constexpr size_t _Mask_64 = ~((static_cast(1) << 6) - 1); + if (_Byte_length(_First1, _Last1) >= 64) { + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_64); + do { + const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); + const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); + const uint8x16_t _Left3 = vld1q_u8(static_cast(_First1) + 32); + const uint8x16_t _Left4 = vld1q_u8(static_cast(_First1) + 48); + const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); + const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); + const uint8x16_t _Right3 = vld1q_u8(static_cast(_First2) + 32); + const uint8x16_t _Right4 = vld1q_u8(static_cast(_First2) + 48); + vst1q_u8(static_cast(_First1) + 0, _Right1); + vst1q_u8(static_cast(_First1) + 16, _Right2); + vst1q_u8(static_cast(_First1) + 32, _Right3); + vst1q_u8(static_cast(_First1) + 48, _Right4); + vst1q_u8(static_cast(_First2) + 0, _Left1); + vst1q_u8(static_cast(_First2) + 16, _Left2); + vst1q_u8(static_cast(_First2) + 32, _Left3); + vst1q_u8(static_cast(_First2) + 48, _Left4); + _Advance_bytes(_First1, 64); + _Advance_bytes(_First2, 64); + } while (_First1 != _Stop_at); + } + + constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); + if (_Byte_length(_First1, _Last1) >= 32) { + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_32); + do { + const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); + const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); + const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); + const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); + vst1q_u8(static_cast(_First1) + 0, _Right1); + vst1q_u8(static_cast(_First1) + 16, _Right2); + vst1q_u8(static_cast(_First2) + 0, _Left1); + vst1q_u8(static_cast(_First2) + 16, _Left2); + _Advance_bytes(_First1, 32); + _Advance_bytes(_First2, 32); + } while (_First1 != _Stop_at); + } + + constexpr size_t _Mask_16 = ~((static_cast(1) << 4) - 1); + if (_Byte_length(_First1, _Last1) >= 16) { + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_16); + do { + const uint8x16_t _Left = vld1q_u8(static_cast(_First1)); + const uint8x16_t _Right = vld1q_u8(static_cast(_First2)); + vst1q_u8(static_cast(_First1), _Right); + vst1q_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 16); + _Advance_bytes(_First2, 16); + } while (_First1 != _Stop_at); + } + + constexpr size_t _Mask_8 = ~((static_cast(1) << 3) - 1); + if (_Byte_length(_First1, _Last1) >= 8) { + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_8); + do { + const uint8x8_t _Left = vld1_u8(static_cast(_First1)); + const uint8x8_t _Right = vld1_u8(static_cast(_First2)); + vst1_u8(static_cast(_First1), _Right); + vst1_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 8); + _Advance_bytes(_First2, 8); + } while (_First1 != _Stop_at); + } + + constexpr size_t _Mask_4 = ~((static_cast(1) << 2) - 1); + if (_Byte_length(_First1, _Last1) >= 4) { + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_4); + do { + uint32x2_t _Left = vdup_n_u32(0); + uint32x2_t _Right = vdup_n_u32(0); + _Left = vld1_lane_u32(static_cast(_First1), _Left, 0); + _Right = vld1_lane_u32(static_cast(_First2), _Right, 0); + vst1_lane_u32(static_cast(_First1), _Right, 0); + vst1_lane_u32(static_cast(_First2), _Left, 0); + _Advance_bytes(_First1, 4); + _Advance_bytes(_First2, 4); + } while (_First1 != _Stop_at); + } + + auto _First1c = static_cast(_First1); + auto _First2c = static_cast(_First2); + for (; _First1c != _Last1; ++_First1c, ++_First2c) { + const unsigned char _Ch = *_First1c; + *_First1c = *_First2c; + *_First2c = _Ch; + } +} +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* const _Last1, void* _First2) noexcept { #ifndef _M_ARM64EC @@ -156,6 +261,7 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( *_First2c = _Ch; } } +#endif // ^^^ !defined(_M_ARM64) // TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility void* __cdecl __std_swap_ranges_trivially_swappable( @@ -166,6 +272,7 @@ void* __cdecl __std_swap_ranges_trivially_swappable( } // extern "C" +#if !defined(_M_ARM64) namespace { namespace _Rotating { void _Swap_3_ranges(void* _First1, void* const _Last1, void* _First2, void* _First3) noexcept { @@ -7694,4 +7801,5 @@ __declspec(noalias) bool __stdcall __std_bitset_from_string_2(void* const _Dest, } } // extern "C" -#endif // defined(_M_IX86) || defined(_M_X64) +#endif // ^^^ !defined(_M_ARM64) +#endif // defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) From 55a37155b04cd71505528f738b5a50807d7ff975 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Fri, 31 Oct 2025 17:59:18 +0000 Subject: [PATCH 2/7] Address whitespace & preprocessor review comments --- stl/src/vector_algorithms.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 8f570215c55..2eb9014bf44 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5,7 +5,7 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) // NB: includes _M_ARM64EC +#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) // NB: _M_X64 includes _M_ARM64EC #include <__msvc_minmax.hpp> #include #include @@ -85,7 +85,6 @@ extern "C" { #ifdef _M_ARM64 __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* const _Last1, void* _First2) noexcept { - constexpr size_t _Mask_64 = ~((static_cast(1) << 6) - 1); if (_Byte_length(_First1, _Last1) >= 64) { const void* _Stop_at = _First1; @@ -261,7 +260,7 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( *_First2c = _Ch; } } -#endif // ^^^ !defined(_M_ARM64) +#endif // ^^^ !defined(_M_ARM64) ^^^ // TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility void* __cdecl __std_swap_ranges_trivially_swappable( @@ -272,7 +271,7 @@ void* __cdecl __std_swap_ranges_trivially_swappable( } // extern "C" -#if !defined(_M_ARM64) +#ifndef _M_ARM64 namespace { namespace _Rotating { void _Swap_3_ranges(void* _First1, void* const _Last1, void* _First2, void* _First3) noexcept { @@ -7801,5 +7800,5 @@ __declspec(noalias) bool __stdcall __std_bitset_from_string_2(void* const _Dest, } } // extern "C" -#endif // ^^^ !defined(_M_ARM64) +#endif // ^^^ !defined(_M_ARM64) ^^^ #endif // defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) From 86896d1b13b3ae231832585f72c6cea812de993f Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Mon, 3 Nov 2025 15:11:26 +0000 Subject: [PATCH 3/7] Remove unecessary ifdef --- stl/src/vector_algorithms.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2eb9014bf44..bfa18678682 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5,7 +5,6 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) // NB: _M_X64 includes _M_ARM64EC #include <__msvc_minmax.hpp> #include #include @@ -7801,4 +7800,3 @@ __declspec(noalias) bool __stdcall __std_bitset_from_string_2(void* const _Dest, } // extern "C" #endif // ^^^ !defined(_M_ARM64) ^^^ -#endif // defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) From 250a17a0d99191237c6f541179735e0e7ac61eae Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 4 Nov 2025 08:27:24 -0800 Subject: [PATCH 4/7] ARM64 doesn't need legacy `__std_swap_ranges_trivially_swappable`. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index bfa18678682..25a06e559c3 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -259,14 +259,14 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( *_First2c = _Ch; } } -#endif // ^^^ !defined(_M_ARM64) ^^^ -// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility +// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility (x64/x86/ARM64EC) void* __cdecl __std_swap_ranges_trivially_swappable( void* const _First1, void* const _Last1, void* const _First2) noexcept { __std_swap_ranges_trivially_swappable_noalias(_First1, _Last1, _First2); return static_cast(_First2) + (static_cast(_Last1) - static_cast(_First1)); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" From a16cb5a1411acb6aaa2ff08365e7464d614b2e90 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 4 Nov 2025 08:43:22 -0800 Subject: [PATCH 5/7] Override /Os for all architectures, before any function defns, cite GH 2108. --- stl/src/vector_algorithms.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 25a06e559c3..66bb3a1a6e7 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5,6 +5,10 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif +#ifndef _DEBUG +#pragma optimize("t", on) // TRANSITION, GH-2108: Override /Os with /Ot for this TU before any function definitions +#endif + #include <__msvc_minmax.hpp> #include #include @@ -16,10 +20,6 @@ #include extern "C" long __isa_enabled; - -#ifndef _DEBUG -#pragma optimize("t", on) // Override /Os with /Ot for this TU -#endif // !defined(_DEBUG) #endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ #ifdef _M_ARM64 From 223967086b4df882a54f90399280c4e0eb6bdb1c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 4 Nov 2025 10:28:28 -0800 Subject: [PATCH 6/7] Improve perf: Only the 64-byte step needs to loop. --- stl/src/vector_algorithms.cpp | 80 +++++++++++++---------------------- 1 file changed, 30 insertions(+), 50 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 66bb3a1a6e7..9072ef0ff52 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -110,66 +110,46 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( } while (_First1 != _Stop_at); } - constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); if (_Byte_length(_First1, _Last1) >= 32) { - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_32); - do { - const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); - const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); - const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); - const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); - vst1q_u8(static_cast(_First1) + 0, _Right1); - vst1q_u8(static_cast(_First1) + 16, _Right2); - vst1q_u8(static_cast(_First2) + 0, _Left1); - vst1q_u8(static_cast(_First2) + 16, _Left2); - _Advance_bytes(_First1, 32); - _Advance_bytes(_First2, 32); - } while (_First1 != _Stop_at); + const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); + const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); + const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); + const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); + vst1q_u8(static_cast(_First1) + 0, _Right1); + vst1q_u8(static_cast(_First1) + 16, _Right2); + vst1q_u8(static_cast(_First2) + 0, _Left1); + vst1q_u8(static_cast(_First2) + 16, _Left2); + _Advance_bytes(_First1, 32); + _Advance_bytes(_First2, 32); } - constexpr size_t _Mask_16 = ~((static_cast(1) << 4) - 1); if (_Byte_length(_First1, _Last1) >= 16) { - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_16); - do { - const uint8x16_t _Left = vld1q_u8(static_cast(_First1)); - const uint8x16_t _Right = vld1q_u8(static_cast(_First2)); - vst1q_u8(static_cast(_First1), _Right); - vst1q_u8(static_cast(_First2), _Left); - _Advance_bytes(_First1, 16); - _Advance_bytes(_First2, 16); - } while (_First1 != _Stop_at); + const uint8x16_t _Left = vld1q_u8(static_cast(_First1)); + const uint8x16_t _Right = vld1q_u8(static_cast(_First2)); + vst1q_u8(static_cast(_First1), _Right); + vst1q_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 16); + _Advance_bytes(_First2, 16); } - constexpr size_t _Mask_8 = ~((static_cast(1) << 3) - 1); if (_Byte_length(_First1, _Last1) >= 8) { - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_8); - do { - const uint8x8_t _Left = vld1_u8(static_cast(_First1)); - const uint8x8_t _Right = vld1_u8(static_cast(_First2)); - vst1_u8(static_cast(_First1), _Right); - vst1_u8(static_cast(_First2), _Left); - _Advance_bytes(_First1, 8); - _Advance_bytes(_First2, 8); - } while (_First1 != _Stop_at); + const uint8x8_t _Left = vld1_u8(static_cast(_First1)); + const uint8x8_t _Right = vld1_u8(static_cast(_First2)); + vst1_u8(static_cast(_First1), _Right); + vst1_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 8); + _Advance_bytes(_First2, 8); } - constexpr size_t _Mask_4 = ~((static_cast(1) << 2) - 1); if (_Byte_length(_First1, _Last1) >= 4) { - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_4); - do { - uint32x2_t _Left = vdup_n_u32(0); - uint32x2_t _Right = vdup_n_u32(0); - _Left = vld1_lane_u32(static_cast(_First1), _Left, 0); - _Right = vld1_lane_u32(static_cast(_First2), _Right, 0); - vst1_lane_u32(static_cast(_First1), _Right, 0); - vst1_lane_u32(static_cast(_First2), _Left, 0); - _Advance_bytes(_First1, 4); - _Advance_bytes(_First2, 4); - } while (_First1 != _Stop_at); + uint32x2_t _Left = vdup_n_u32(0); + uint32x2_t _Right = vdup_n_u32(0); + _Left = vld1_lane_u32(static_cast(_First1), _Left, 0); + _Right = vld1_lane_u32(static_cast(_First2), _Right, 0); + vst1_lane_u32(static_cast(_First1), _Right, 0); + vst1_lane_u32(static_cast(_First2), _Left, 0); + _Advance_bytes(_First1, 4); + _Advance_bytes(_First2, 4); } auto _First1c = static_cast(_First1); From 3a42a4fb5b08a78dc4a76383a548fd6c98d7b783 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 4 Nov 2025 10:34:43 -0800 Subject: [PATCH 7/7] Reduce the scope of `_Mask_64`. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 9072ef0ff52..ac2d743a32d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -84,9 +84,9 @@ extern "C" { #ifdef _M_ARM64 __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* const _Last1, void* _First2) noexcept { - constexpr size_t _Mask_64 = ~((static_cast(1) << 6) - 1); if (_Byte_length(_First1, _Last1) >= 64) { - const void* _Stop_at = _First1; + constexpr size_t _Mask_64 = ~((static_cast(1) << 6) - 1); + const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_64); do { const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0);