From 51f0eeefacea559ab7e74292dcccab31989aefb4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 11 May 2025 13:07:05 +0300 Subject: [PATCH 1/6] benchmark --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/reverse.cpp | 53 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 benchmarks/src/reverse.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 04e47486df5..0a9074781e9 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -118,6 +118,7 @@ add_benchmark(priority_queue_push_range src/priority_queue_push_range.cpp) add_benchmark(random_integer_generation src/random_integer_generation.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) +add_benchmark(reverse src/reverse.cpp) add_benchmark(search src/search.cpp) add_benchmark(search_n src/search_n.cpp) add_benchmark(std_copy src/std_copy.cpp) diff --git a/benchmarks/src/reverse.cpp b/benchmarks/src/reverse.cpp new file mode 100644 index 00000000000..b4566a9aaa1 --- /dev/null +++ b/benchmarks/src/reverse.cpp @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#include "skewed_allocator.hpp" +#include "utility.hpp" + +template +void r(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + auto v = random_vector(size); + + for (auto _ : state) { + benchmark::DoNotOptimize(v); + std::reverse(v.begin(), v.end()); + } +} + +template +void rc(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + auto v = random_vector(size); + std::vector> d(size); + + for (auto _ : state) { + benchmark::DoNotOptimize(v); + std::reverse_copy(v.begin(), v.end(), d.begin()); + benchmark::DoNotOptimize(d); + } +} + +void common_args(auto bm) { + bm->Arg(3449); + // AVX tail tests + bm->Arg(63)->Arg(31)->Arg(15)->Arg(7); +} + + +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); + +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); + +BENCHMARK_MAIN(); From ef32baaf1cd5608acc2bccd8609abf9e87a8d2b9 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 11 May 2025 13:07:36 +0300 Subject: [PATCH 2/6] traits --- stl/src/vector_algorithms.cpp | 415 +++++++++++----------------------- 1 file changed, 132 insertions(+), 283 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0fb916c1318..3f92ddc3973 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -165,8 +165,61 @@ void* __cdecl __std_swap_ranges_trivially_swappable( namespace { namespace _Reversing { + struct _Traits_1 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + + const __m256i _Perm = _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + return _mm256_shuffle_epi8(_Perm, _Reverse_char_lanes_avx); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return _mm_shuffle_epi8(_Val, _Reverse_char_sse); + } + }; + + struct _Traits_2 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // + 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // + 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + + const __m256i _Perm = _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + return _mm256_shuffle_epi8(_Perm, _Reverse_short_lanes_avx); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return _mm_shuffle_epi8(_Val, _Reverse_short_sse); + } + }; + + struct _Traits_4 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + return _mm256_permutevar8x32_epi32(_Val, _Shuf); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + return _mm_shuffle_epi32(_Val, _MM_SHUFFLE(0, 1, 2, 3)); + } + }; + + struct _Traits_8 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(0, 1, 2, 3)); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + return _mm_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } + }; + template - void _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { + void __stdcall _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { for (; _First != _Last && _First != --_Last; ++_First) { const auto _Temp = *_First; *_First = *_Last; @@ -180,323 +233,119 @@ namespace { *_Dest++ = *--_Last; } } - } // namespace _Reversing -} // unnamed namespace - -extern "C" { -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { + template + __declspec(noalias) void __cdecl _Reverse_impl(void* _First, void* _Last) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - // vpermq to load left and right, and transpose the lanes - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Right_perm = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - // transpose all the chars in the lanes - const __m256i _Left_reversed = _mm256_shuffle_epi8(_Left_perm, _Reverse_char_lanes_avx); - const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_char_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); + if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); + do { + _Advance_bytes(_Last, -32); + const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); + const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); + const __m256i _Left_reversed = _Traits::_Rev_avx(_Left); + const __m256i _Right_reversed = _Traits::_Rev_avx(_Right); + _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); + _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); - const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } + if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); + do { + _Advance_bytes(_Last, -16); + const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); + const __m128i _Left_reversed = _Traits::_Rev_sse(_Left); + const __m128i _Right_reversed = _Traits::_Rev_sse(_Right); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); + _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); + _Advance_bytes(_First, 16); + } while (_First != _Stop_at); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); -} + _Reverse_tail(static_cast<_Ty*>(_First), static_cast<_Ty*>(_Last)); + } -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { + template + __declspec(noalias) void __cdecl _Reverse_copy_impl( + const void* _First, const void* _Last, void* _Dest) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Right_perm = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Left_reversed = _mm256_shuffle_epi8(_Left_perm, _Reverse_short_lanes_avx); - const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_short_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); + if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); + do { + _Advance_bytes(_Last, -32); + const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); + const __m256i _Block_reversed = _Traits::_Rev_avx(_Block); + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); + _Advance_bytes(_Dest, 32); + } while (_Dest != _Stop_at); - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); - const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } + if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); + do { + _Advance_bytes(_Last, -16); + const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); + const __m128i _Block_reversed = _Traits::_Rev_sse(_Block); + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); + _Advance_bytes(_Dest, 16); + } while (_Dest != _Stop_at); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); -} + _Reverse_copy_tail( + static_cast(_First), static_cast(_Last), static_cast<_Ty*>(_Dest)); + } + } // namespace _Reversing +} // unnamed namespace -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_reversed = _mm256_permutevar8x32_epi32(_Left, _Shuf); - const __m256i _Right_reversed = _mm256_permutevar8x32_epi32(_Right, _Shuf); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); +extern "C" { - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_1, uint8_t>(_First, _Last); +} - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(0, 1, 2, 3)); - const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(0, 1, 2, 3)); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_2, uint16_t>(_First, _Last); +} - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_4, uint32_t>(_First, _Last); } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_reversed = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(0, 1, 2, 3)); - const __m256i _Right_reversed = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(0, 1, 2, 3)); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); + _Reversing::_Reverse_impl<_Reversing::_Traits_8, uint64_t>(_First, _Last); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_char_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), static_cast(_Last), - static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_1, uint8_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_short_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), - static_cast(_Last), static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_2, uint16_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_reversed = _mm256_permutevar8x32_epi32(_Block, _Shuf); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(0, 1, 2, 3)); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), static_cast(_Last), - static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_4, uint32_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_reversed = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(0, 1, 2, 3)); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), - static_cast(_Last), static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_8, uint64_t>(_First, _Last, _Dest); } } // extern "C" From 79226ff89aa4b4650494218187595a3ff5305108 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 11 May 2025 14:24:47 +0300 Subject: [PATCH 3/6] avx2 tail --- stl/src/vector_algorithms.cpp | 65 +++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 3f92ddc3973..f70fd628d6a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -234,12 +234,22 @@ namespace { } } +#ifndef _M_ARM64EC + __m256i _Avx2_rev_tail_mask_32(const size_t _Count_in_bytes) noexcept { + // _Count_in_bytes must be within [0, 32]. + static constexpr unsigned int _Tail_masks[16] = { + 0, 0, 0, 0, 0, 0, 0, 0, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u}; + return _mm256_loadu_si256(reinterpret_cast( + reinterpret_cast(_Tail_masks) + _Count_in_bytes)); + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + template __declspec(noalias) void __cdecl _Reverse_impl(void* _First, void* _Last) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 64 && _Use_avx2()) { const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); + _Advance_bytes(_Stop_at, (_Length >> 1) & ~size_t{0x1F}); do { _Advance_bytes(_Last, -32); const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); @@ -251,12 +261,32 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); + constexpr size_t _Tail_elem_mask = 0x1C & ~size_t{sizeof(_Ty) - 1}; + + if (const size_t _Avx_tail = (_Length >> 1) & _Tail_elem_mask; _Avx_tail != 0) { + _Advance_bytes(_Last, -32); + const __m256i _Mask = _Avx2_tail_mask_32(_Avx_tail); + const __m256i _Rev_mask = _Avx2_rev_tail_mask_32(_Avx_tail); + const __m256i _Left = _mm256_maskload_epi32(static_cast(_First), _Mask); + const __m256i _Right = _mm256_maskload_epi32(static_cast(_Last), _Rev_mask); + const __m256i _Left_reversed = _Traits::_Rev_avx(_Left); + const __m256i _Right_reversed = _Traits::_Rev_avx(_Right); + _mm256_maskstore_epi32(static_cast(_First), _Mask, _Right_reversed); + _mm256_maskstore_epi32(static_cast(_Last), _Rev_mask, _Left_reversed); + if constexpr (sizeof(_Ty) < 4) { + _Advance_bytes(_First, _Avx_tail); + _Advance_bytes(_Last, 32 - _Avx_tail); + } + } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (_Length >= 32 && _Use_sse42()) { const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); + _Advance_bytes(_Stop_at, (_Length >> 1) & ~size_t{0xF}); do { _Advance_bytes(_Last, -16); const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); @@ -277,9 +307,9 @@ namespace { __declspec(noalias) void __cdecl _Reverse_copy_impl( const void* _First, const void* _Last, void* _Dest) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 32 && _Use_avx2()) { const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); + _Advance_bytes(_Stop_at, _Length & ~size_t{0x1F}); do { _Advance_bytes(_Last, -32); const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); @@ -288,12 +318,27 @@ namespace { _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); + if (const size_t _Avx_tail = _Length & 0x1C; _Avx_tail != 0) { + _Advance_bytes(_Last, -32); + const __m256i _Mask = _Avx2_tail_mask_32(_Avx_tail); + const __m256i _Rev_mask = _Avx2_rev_tail_mask_32(_Avx_tail); + const __m256i _Block = _mm256_maskload_epi32(static_cast(_Last), _Rev_mask); + const __m256i _Block_reversed = _Traits::_Rev_avx(_Block); + _mm256_maskstore_epi32(static_cast(_Dest), _Mask, _Block_reversed); + if constexpr (sizeof(_Ty) < 4) { + _Advance_bytes(_Dest, _Avx_tail); + _Advance_bytes(_Last, 32 - _Avx_tail); + } + } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (_Length >= 16 && _Use_sse42()) { const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); + _Advance_bytes(_Stop_at, _Length & ~size_t{0xF}); do { _Advance_bytes(_Last, -16); const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); From d343d0378190c48ea635d66fc3fddca896a4b633 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 11 May 2025 14:34:15 +0300 Subject: [PATCH 4/6] undo avx2 tail for non-copy version --- stl/src/vector_algorithms.cpp | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f70fd628d6a..33cb4e5f692 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -261,30 +261,10 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); - constexpr size_t _Tail_elem_mask = 0x1C & ~size_t{sizeof(_Ty) - 1}; - - if (const size_t _Avx_tail = (_Length >> 1) & _Tail_elem_mask; _Avx_tail != 0) { - _Advance_bytes(_Last, -32); - const __m256i _Mask = _Avx2_tail_mask_32(_Avx_tail); - const __m256i _Rev_mask = _Avx2_rev_tail_mask_32(_Avx_tail); - const __m256i _Left = _mm256_maskload_epi32(static_cast(_First), _Mask); - const __m256i _Right = _mm256_maskload_epi32(static_cast(_Last), _Rev_mask); - const __m256i _Left_reversed = _Traits::_Rev_avx(_Left); - const __m256i _Right_reversed = _Traits::_Rev_avx(_Right); - _mm256_maskstore_epi32(static_cast(_First), _Mask, _Right_reversed); - _mm256_maskstore_epi32(static_cast(_Last), _Rev_mask, _Left_reversed); - if constexpr (sizeof(_Ty) < 4) { - _Advance_bytes(_First, _Avx_tail); - _Advance_bytes(_Last, 32 - _Avx_tail); - } - } - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } - if constexpr (sizeof(_Ty) >= 4) { - return; - } - } else if (_Length >= 32 && _Use_sse42()) { + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 32 && _Use_sse42()) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, (_Length >> 1) & ~size_t{0xF}); do { From 70724d4bb8494fe2c8778fff2fa75aea4ad9d70d Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 11 May 2025 15:05:57 +0300 Subject: [PATCH 5/6] stray __stdcat --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 33cb4e5f692..2f58438821b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -219,7 +219,7 @@ namespace { }; template - void __stdcall _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { + void _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { for (; _First != _Last && _First != --_Last; ++_First) { const auto _Temp = *_First; *_First = *_Last; From b642282309a9b9125496924e4b2955a5a83c2a9c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 15 May 2025 08:17:10 -0700 Subject: [PATCH 6/6] Add ARM64EC guards. --- stl/src/vector_algorithms.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2f58438821b..1574a876380 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -165,6 +165,12 @@ void* __cdecl __std_swap_ranges_trivially_swappable( namespace { namespace _Reversing { +#ifdef _M_ARM64EC + using _Traits_1 = void; + using _Traits_2 = void; + using _Traits_4 = void; + using _Traits_8 = void; +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv struct _Traits_1 { static __m256i _Rev_avx(const __m256i _Val) noexcept { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // @@ -217,6 +223,7 @@ namespace { return _mm_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); } }; +#endif // ^^^ !defined(_M_ARM64EC) ^^^ template void _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept {