From 28fbeeea46734bf7d10b619002380b26ef7449da Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 13 Apr 2024 23:12:28 +0300 Subject: [PATCH 01/23] Vectorize `find_first_of` for 4 and 8 byte elements --- benchmarks/src/find_first_of.cpp | 22 ++- stl/inc/algorithm | 15 +- stl/src/vector_algorithms.cpp | 255 +++++++++++++++++++++++++++++-- 3 files changed, 261 insertions(+), 31 deletions(-) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp index 4697ccd3c8a..05731a18938 100644 --- a/benchmarks/src/find_first_of.cpp +++ b/benchmarks/src/find_first_of.cpp @@ -33,18 +33,14 @@ void bm(benchmark::State& state) { } } -#define ARGS \ - Args({2, 3}) \ - ->Args({7, 4}) \ - ->Args({9, 3}) \ - ->Args({22, 5}) \ - ->Args({58, 2}) \ - ->Args({102, 4}) \ - ->Args({325, 1}) \ - ->Args({1011, 11}) \ - ->Args({3056, 7}); - -BENCHMARK(bm)->ARGS; -BENCHMARK(bm)->ARGS; +void common_args(auto bm) { + bm->Args({2, 3})->Args({7, 4})->Args({9, 3})->Args({22, 5})->Args({58, 2}); + bm->Args({102, 4})->Args({325, 1})->Args({1011, 11})->Args({3056, 7}); +} + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index df17fea0dba..a9c776a08e3 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -62,6 +62,11 @@ const void* __stdcall __std_find_first_of_trivial_1( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; const void* __stdcall __std_find_first_of_trivial_2( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; +const void* __stdcall __std_find_first_of_trivial_4( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; +const void* __stdcall __std_find_first_of_trivial_8( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; + __declspec(noalias) _Min_max_1i __stdcall __std_minmax_1i(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_1u __stdcall __std_minmax_1u(const void* _First, const void* _Last) noexcept; @@ -202,6 +207,12 @@ _Ty1* _Find_first_of_vectorized( } else if constexpr (sizeof(_Ty1) == 2) { return const_cast<_Ty1*>( static_cast(::__std_find_first_of_trivial_2(_First1, _Last1, _First2, _Last2))); + } else if constexpr (sizeof(_Ty1) == 4) { + return const_cast<_Ty1*>( + static_cast(::__std_find_first_of_trivial_4(_First1, _Last1, _First2, _Last2))); + } else if constexpr (sizeof(_Ty1) == 8) { + return const_cast<_Ty1*>( + static_cast(::__std_find_first_of_trivial_8(_First1, _Last1, _First2, _Last2))); } else { static_assert(_Always_false<_Ty1>, "Unexpected size"); } @@ -230,9 +241,7 @@ _INLINE_VAR constexpr ptrdiff_t _Threshold_find_first_of = 16; // Can we activate the vector algorithms for find_first_of? template -constexpr bool _Vector_alg_in_find_first_of_is_safe = - _Equal_memcmp_is_safe<_It1, _It2, _Pr> // can replace value comparison with bitwise comparison - && sizeof(_Iter_value_t<_It1>) <= 2; // pcmpestri compatible size +constexpr bool _Vector_alg_in_find_first_of_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr>; // Can we activate the vector algorithms for replace? template diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2349d804e3f..2f8eeb0f653 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -42,9 +42,10 @@ namespace { }; __m256i _Avx2_tail_mask_32(const size_t _Count_in_dwords) noexcept { - // _Count_in_dwords must be within [1, 7]. - static constexpr unsigned int _Tail_masks[14] = {~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, 0, 0, 0, 0, 0, 0, 0}; - return _mm256_loadu_si256(reinterpret_cast(_Tail_masks + (7 - _Count_in_dwords))); + // _Count_in_dwords must be within [0, 8]. + static constexpr unsigned int _Tail_masks[16] = { + ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, 0, 0, 0, 0, 0, 0, 0, 0}; + return _mm256_loadu_si256(reinterpret_cast(_Tail_masks + (8 - _Count_in_dwords))); } } // namespace #endif // !defined(_M_ARM64EC) @@ -2038,7 +2039,26 @@ namespace { } template - const void* __stdcall __std_find_first_of_trivial_impl( + const void* __stdcall __std_find_first_of_trivial_fallback( + const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) { + auto _Ptr_haystack = static_cast(_First1); + const auto _Ptr_haystack_end = static_cast(_Last1); + const auto _Ptr_needle = static_cast(_First2); + const auto _Ptr_needle_end = static_cast(_Last2); + + for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { + for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { + if (*_Ptr_haystack == *_Ptr) { + return _Ptr_haystack; + } + } + } + + return _Ptr_haystack; + } + + template + const void* __stdcall __std_find_first_of_trivial_pcmpestri_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC if (_Use_sse42()) { @@ -2175,21 +2195,216 @@ namespace { } } #endif // !_M_ARM64EC + return __std_find_first_of_trivial_fallback<_Ty>(_First1, _Last1, _First2, _Last2); + } - auto _Ptr_haystack = static_cast(_First1); - const auto _Ptr_haystack_end = static_cast(_Last1); - const auto _Ptr_needle = static_cast(_First2); - const auto _Ptr_needle_end = static_cast(_Last2); + struct _Find_first_of_traits_4 : _Find_traits_4 { + using _Ty = uint32_t; + + template + static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { + if constexpr (_Amount == 1) { + return _mm256_broadcastd_epi32(_mm256_castsi256_si128(_Val)); + } else if constexpr (_Amount == 2) { + return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); + } else if constexpr (_Amount == 4) { + if (_Needle_length_el < 4) { + _Val = _mm256_insert_epi32(_Val, _mm256_cvtsi256_si32(_Val), 3); + } - for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { - for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { - if (*_Ptr_haystack == *_Ptr) { - return _Ptr_haystack; + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); + } else if constexpr (_Amount == 8) { + if (_Needle_length_el < _Amount) { + const __m256i _Mask = _Avx2_tail_mask_32(_Needle_length_el); + // zero unused elements in sequenctial permutation mask, so will be filled by 1st + const __m256i _Perm = _mm256_and_si256(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), _Mask); + _Val = _mm256_permutevar8x32_epi32(_Val, _Perm); } + + return _Val; + } else { + static_assert(_Amount != _Amount, "Unexpected amount"); } } - return _Ptr_haystack; + template + static __m256i _Shuffle_avx(const __m256i _Val) noexcept { + if constexpr (_Amount == 1) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(2, 3, 0, 1)); + } else if constexpr (_Amount == 2) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else if constexpr (_Amount == 4) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else { + static_assert(_Amount != _Amount, "Unexpected amount"); + } + } + }; + + struct _Find_first_of_traits_8 : _Find_traits_8 { + using _Ty = uint64_t; + + template + static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { + if constexpr (_Amount == 1) { + return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); + } else if constexpr(_Amount == 2) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); + } else if constexpr (_Amount == 4) { + if (_Needle_length_el < 4) { + _Val = _mm256_insert_epi64(_Val, _mm256_cvtsi256_si64(_Val), 3); + } + + return _Val; + } else { + static_assert(_Amount != _Amount, "Unexpected amount"); + } + } + + template + static __m256i _Shuffle_avx(const __m256i _Val) noexcept { + if constexpr (_Amount == 1) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else if constexpr (_Amount == 2) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else { + static_assert(_Amount != _Amount, "Unexpected amount"); + } + } + }; + + template + const __m256i __std_find_first_of_trivial_shuffle_step(const __m256i _Data1, const __m256i _Data2s0) { + __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); + if constexpr (_Needle_length_el_magnitude >= 2) { + const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s1)); + if constexpr (_Needle_length_el_magnitude >= 4) { + const __m256i _Data2s2 = _Traits::_Shuffle_avx<2>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s2)); + const __m256i _Data2s3 = _Traits::_Shuffle_avx<1>(_Data2s2); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s3)); + if constexpr (_Needle_length_el_magnitude >= 8) { + const __m256i _Data2s4 = _Traits::_Shuffle_avx<4>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s4)); + const __m256i _Data2s5 = _Traits::_Shuffle_avx<1>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s5)); + const __m256i _Data2s6 = _Traits::_Shuffle_avx<2>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s6)); + const __m256i _Data2s7 = _Traits::_Shuffle_avx<1>(_Data2s6); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s7)); + } + } + } + return _Eq; + } + + template + const void* __std_find_first_of_trivial_shuffle_impl(const void* _First1, const void* const _Last1, + const void* const _First2, const size_t _Needle_length_el) { + using _Ty = typename _Traits::_Ty; + const __m256i _Data2 = _mm256_maskload_epi32(reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); + const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Haystack_length & ~size_t{0x1F}); + + for (; _First1 != _Stop1; _Advance_bytes(_First1, 32)) { + const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); + const __m256i _Eq = + __std_find_first_of_trivial_shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const int _Bingo = _mm256_movemask_epi8(_Eq); + + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First1, _Offset); + return _First1; + } + } + + if (const size_t _Haystack_tail_length = _Haystack_length & 0x1C; _Haystack_tail_length != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Haystack_tail_length >> 2); + const __m256i _Data1 = _mm256_maskload_epi32(static_cast(_First1), _Tail_mask); + const __m256i _Eq = + __std_find_first_of_trivial_shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); + + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First1, _Offset); + return _First1; + } + + _Advance_bytes(_First1, _Haystack_tail_length); + } + + return _First1; + } + + template + const void* __stdcall __std_find_first_of_trivial_48_impl( + const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { + using _Ty = typename _Traits::_Ty; +#ifndef _M_ARM64EC + if (_Use_avx2()) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 + + const size_t _Needle_length = _Byte_length(_First2, _Last2); + const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); + + // Special handling of small needle + // The generic approach could also handle it but with worse performance + if (_Needle_length_el == 0) { + return _Last1; + } else if (_Needle_length_el == 1) { + // This is expected to be done on an upper level with better efficeiency + return __std_find_first_of_trivial_shuffle_impl<_Traits, 1>( + _First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el == 2) { + return __std_find_first_of_trivial_shuffle_impl<_Traits, 2>( + _First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el <= 4) { + return __std_find_first_of_trivial_shuffle_impl<_Traits, 4>( + _First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el <= 8) { + if constexpr (sizeof(_Ty) == 4) { + return __std_find_first_of_trivial_shuffle_impl<_Traits, 8>( + _First1, _Last1, _First2, _Needle_length_el); + } + } + + // Generic approach + const size_t _Needle_length_tail = _Needle_length & 0x1C; + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Needle_length_tail >> 2); + + const void* _Stop2 = _First2; + _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); + + for (auto _Ptr1 = static_cast(_First1); _Ptr1 != _Last1; ++_Ptr1) { + const auto _Data1 = _Traits::_Set_avx(*_Ptr1); + for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { + const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); + const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); + if (!_mm256_testz_si256(_Eq, _Eq)) { + return _Ptr1; + } + } + + if (_Needle_length_tail != 0) { + const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); + const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); + if (!_mm256_testz_si256(_Eq, _Tail_mask)) { + return _Ptr1; + } + } + } + + return _Last1; + } +#endif // !_M_ARM64EC + return __std_find_first_of_trivial_fallback<_Ty>(_First1, _Last1, _First2, _Last2); } @@ -2352,12 +2567,22 @@ __declspec(noalias) size_t const void* __stdcall __std_find_first_of_trivial_1( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of_trivial_pcmpestri_impl(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_2( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of_trivial_pcmpestri_impl(_First1, _Last1, _First2, _Last2); +} + +const void* __stdcall __std_find_first_of_trivial_4( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { + return __std_find_first_of_trivial_48_impl<_Find_first_of_traits_4>(_First1, _Last1, _First2, _Last2); +} + +const void* __stdcall __std_find_first_of_trivial_8( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { + return __std_find_first_of_trivial_48_impl<_Find_first_of_traits_8>(_First1, _Last1, _First2, _Last2); } __declspec(noalias) size_t From 721bfed332a34a519f90c99d0d76a103a87742ba Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 13 Apr 2024 23:33:42 +0300 Subject: [PATCH 02/23] Format --- stl/src/vector_algorithms.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2f8eeb0f653..d4ce6dc0516 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2227,7 +2227,7 @@ namespace { } } - template + template static __m256i _Shuffle_avx(const __m256i _Val) noexcept { if constexpr (_Amount == 1) { return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(2, 3, 0, 1)); @@ -2248,7 +2248,7 @@ namespace { static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { if constexpr (_Amount == 1) { return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); - } else if constexpr(_Amount == 2) { + } else if constexpr (_Amount == 2) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); } else if constexpr (_Amount == 4) { if (_Needle_length_el < 4) { @@ -2300,10 +2300,11 @@ namespace { } template - const void* __std_find_first_of_trivial_shuffle_impl(const void* _First1, const void* const _Last1, - const void* const _First2, const size_t _Needle_length_el) { + const void* __std_find_first_of_trivial_shuffle_impl( + const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) { using _Ty = typename _Traits::_Ty; - const __m256i _Data2 = _mm256_maskload_epi32(reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); + const __m256i _Data2 = _mm256_maskload_epi32( + reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); const size_t _Haystack_length = _Byte_length(_First1, _Last1); @@ -2344,8 +2345,8 @@ namespace { } template - const void* __stdcall __std_find_first_of_trivial_48_impl( - const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { + const void* __stdcall __std_find_first_of_trivial_48_impl(const void* const _First1, const void* const _Last1, + const void* const _First2, const void* const _Last2) noexcept { using _Ty = typename _Traits::_Ty; #ifndef _M_ARM64EC if (_Use_avx2()) { From 12e08b403bacaff1ce7814244925ed5490ac0edb Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 00:04:24 +0300 Subject: [PATCH 03/23] fix x86 build --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index d4ce6dc0516..b12a1a8e90d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2209,7 +2209,7 @@ namespace { return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); } else if constexpr (_Amount == 4) { if (_Needle_length_el < 4) { - _Val = _mm256_insert_epi32(_Val, _mm256_cvtsi256_si32(_Val), 3); + _Val = _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(0, 2, 1, 0)); } return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); @@ -2252,7 +2252,7 @@ namespace { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); } else if constexpr (_Amount == 4) { if (_Needle_length_el < 4) { - _Val = _mm256_insert_epi64(_Val, _mm256_cvtsi256_si64(_Val), 3); + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(0, 2, 1, 0)); } return _Val; From 7edceb78a50d75e14fee006dddea4e4cfdbf4c0f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 08:00:28 +0300 Subject: [PATCH 04/23] We don't actually need dependent `false` --- stl/src/vector_algorithms.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b12a1a8e90d..fac3e170af6 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2223,7 +2223,7 @@ namespace { return _Val; } else { - static_assert(_Amount != _Amount, "Unexpected amount"); + static_assert(false, "Unexpected amount"); } } @@ -2236,7 +2236,7 @@ namespace { } else if constexpr (_Amount == 4) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); } else { - static_assert(_Amount != _Amount, "Unexpected amount"); + static_assert(false, "Unexpected amount"); } } }; @@ -2257,7 +2257,7 @@ namespace { return _Val; } else { - static_assert(_Amount != _Amount, "Unexpected amount"); + static_assert(false, "Unexpected amount"); } } @@ -2268,7 +2268,7 @@ namespace { } else if constexpr (_Amount == 2) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); } else { - static_assert(_Amount != _Amount, "Unexpected amount"); + static_assert(false, "Unexpected amount"); } } }; From 3963bea7490b857905223ad137dd0c291b0e0c0f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 10:14:42 +0300 Subject: [PATCH 05/23] Namespace and some renames avoid wrapping --- stl/src/vector_algorithms.cpp | 581 +++++++++++++++++----------------- 1 file changed, 288 insertions(+), 293 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index fac3e170af6..b03840d97b5 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2038,102 +2038,143 @@ namespace { return _Result; } - template - const void* __stdcall __std_find_first_of_trivial_fallback( - const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) { - auto _Ptr_haystack = static_cast(_First1); - const auto _Ptr_haystack_end = static_cast(_Last1); - const auto _Ptr_needle = static_cast(_First2); - const auto _Ptr_needle_end = static_cast(_Last2); - - for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { - for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { - if (*_Ptr_haystack == *_Ptr) { - return _Ptr_haystack; + namespace __std_find_first_of { + + template + const void* __stdcall __fallback( + const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) { + auto _Ptr_haystack = static_cast(_First1); + const auto _Ptr_haystack_end = static_cast(_Last1); + const auto _Ptr_needle = static_cast(_First2); + const auto _Ptr_needle_end = static_cast(_Last2); + + for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { + for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { + if (*_Ptr_haystack == *_Ptr) { + return _Ptr_haystack; + } } } - } - return _Ptr_haystack; - } + return _Ptr_haystack; + } - template - const void* __stdcall __std_find_first_of_trivial_pcmpestri_impl( - const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { + template + const void* __stdcall __pcmpestri_impl(const void* _First1, const void* const _Last1, + const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC - if (_Use_sse42()) { - constexpr int _Op = - (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; - constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - const size_t _Needle_length = _Byte_length(_First2, _Last2); - - if (_Needle_length <= 16) { - // Special handling of small needle - // The generic branch could also handle it but with slightly worse performance + if (_Use_sse42()) { + constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY + | _SIDD_LEAST_SIGNIFICANT; + constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; + const size_t _Needle_length = _Byte_length(_First2, _Last2); + + if (_Needle_length <= 16) { + // Special handling of small needle + // The generic branch could also handle it but with slightly worse performance + + const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First2, _Needle_length); + const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp1)); + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + + while (_First1 != _Stop_at) { + const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); + if (_mm_cmpestrc(_Data2, _Needle_length_el, _Data1, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Needle_length_el, _Data1, _Part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + return _First1; + } - const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); + _Advance_bytes(_First1, 16); + } - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _First2, _Needle_length); - const __m128i _Needle = _mm_load_si128(reinterpret_cast(_Tmp1)); + const size_t _Last_part_size = _Haystack_length & 0xF; + const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); - const size_t _Haystack_length = _Byte_length(_First1, _Last1); - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First1, _Last_part_size); + const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp2)); - while (_First1 != _Stop_at) { - const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); + if (_mm_cmpestrc(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op); _Advance_bytes(_First1, _Pos * sizeof(_Ty)); return _First1; } - _Advance_bytes(_First1, 16); - } + _Advance_bytes(_First1, _Last_part_size); + return _First1; + } else { + const void* _Last_needle = _First2; + _Advance_bytes(_Last_needle, _Needle_length & ~size_t{0xF}); - const size_t _Last_part_size = _Haystack_length & 0xF; - const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); + const int _Last_needle_length = static_cast(_Needle_length & 0xF); - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Haystack_part = _mm_load_si128(reinterpret_cast(_Tmp2)); + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _Last_needle, _Last_needle_length); + const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); + const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op); - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); - return _First1; - } + constexpr int _Not_found = 16; // arbitrary value greater than any found value - _Advance_bytes(_First1, _Last_part_size); - return _First1; - } else { - const void* _Last_needle = _First2; - _Advance_bytes(_Last_needle, _Needle_length & ~size_t{0xF}); + int _Found_pos = _Not_found; + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + + while (_First1 != _Stop_at) { + const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); - const int _Last_needle_length = static_cast(_Needle_length & 0xF); + for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; + _Advance_bytes(_Cur_needle, 16)) { + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + if (_mm_cmpestrc(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } + + if (const int _Needle_length_el = _Last_needle_length_el; _Needle_length_el != 0) { + const __m128i _Data2 = _Last_needle_val; + if (_mm_cmpestrc(_Data2, _Needle_length_el, _Data1, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Needle_length_el, _Data1, _Part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _Last_needle, _Last_needle_length); - const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); - const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); + if (_Found_pos != _Not_found) { + _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); + return _First1; + } - constexpr int _Not_found = 16; // arbitrary value greater than any found value + _Advance_bytes(_First1, 16); + } - int _Found_pos = _Not_found; + const size_t _Last_part_size = _Haystack_length & 0xF; + const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); - const size_t _Haystack_length = _Byte_length(_First1, _Last1); - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First1, _Last_part_size); + const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp2)); - while (_First1 != _Stop_at) { - const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + _Found_pos = _Last_part_size_el; for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { - const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); - if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op); + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + + if (_mm_cmpestrc(_Data2, _Part_size_el, _Data1, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Part_size_el, _Data1, _Last_part_size_el, _Op); if (_Pos < _Found_pos) { _Found_pos = _Pos; } @@ -2141,273 +2182,227 @@ namespace { } if (const int _Needle_length_el = _Last_needle_length_el; _Needle_length_el != 0) { - const __m128i _Needle = _Last_needle_val; - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { - const int _Pos = - _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); + const __m128i _Data2 = _Last_needle_val; + if (_mm_cmpestrc(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op); if (_Pos < _Found_pos) { _Found_pos = _Pos; } } } - if (_Found_pos != _Not_found) { - _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); - return _First1; - } - - _Advance_bytes(_First1, 16); + _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); + return _First1; } + } +#endif // !_M_ARM64EC + return __fallback<_Ty>(_First1, _Last1, _First2, _Last2); + } - const size_t _Last_part_size = _Haystack_length & 0xF; - const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); - - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Haystack_part = _mm_load_si128(reinterpret_cast(_Tmp2)); - - _Found_pos = _Last_part_size_el; - - for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { - const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + struct _Traits_4 : _Find_traits_4 { + using _Ty = uint32_t; - if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Last_part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Last_part_size_el, _Op); - if (_Pos < _Found_pos) { - _Found_pos = _Pos; - } + template + static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { + if constexpr (_Amount == 1) { + return _mm256_broadcastd_epi32(_mm256_castsi256_si128(_Val)); + } else if constexpr (_Amount == 2) { + return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); + } else if constexpr (_Amount == 4) { + if (_Needle_length_el < 4) { + _Val = _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(0, 2, 1, 0)); } - } - if (const int _Needle_length_el = _Last_needle_length_el; _Needle_length_el != 0) { - const __m128i _Needle = _Last_needle_val; - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op)) { - const int _Pos = - _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op); - if (_Pos < _Found_pos) { - _Found_pos = _Pos; - } + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); + } else if constexpr (_Amount == 8) { + if (_Needle_length_el < _Amount) { + const __m256i _Mask = _Avx2_tail_mask_32(_Needle_length_el); + // zero unused elements in sequenctial permutation mask, so will be filled by 1st + const __m256i _Perm = _mm256_and_si256(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), _Mask); + _Val = _mm256_permutevar8x32_epi32(_Val, _Perm); } - } - _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); - return _First1; + return _Val; + } else { + static_assert(false, "Unexpected amount"); + } } - } -#endif // !_M_ARM64EC - return __std_find_first_of_trivial_fallback<_Ty>(_First1, _Last1, _First2, _Last2); - } - struct _Find_first_of_traits_4 : _Find_traits_4 { - using _Ty = uint32_t; - - template - static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { - if constexpr (_Amount == 1) { - return _mm256_broadcastd_epi32(_mm256_castsi256_si128(_Val)); - } else if constexpr (_Amount == 2) { - return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); - } else if constexpr (_Amount == 4) { - if (_Needle_length_el < 4) { - _Val = _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(0, 2, 1, 0)); + template + static __m256i _Shuffle_avx(const __m256i _Val) noexcept { + if constexpr (_Amount == 1) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(2, 3, 0, 1)); + } else if constexpr (_Amount == 2) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else if constexpr (_Amount == 4) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else { + static_assert(false, "Unexpected amount"); } + } + }; + + struct _Traits_8 : _Find_traits_8 { + using _Ty = uint64_t; + + template + static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { + if constexpr (_Amount == 1) { + return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); + } else if constexpr (_Amount == 2) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); + } else if constexpr (_Amount == 4) { + if (_Needle_length_el < 4) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(0, 2, 1, 0)); + } - return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); - } else if constexpr (_Amount == 8) { - if (_Needle_length_el < _Amount) { - const __m256i _Mask = _Avx2_tail_mask_32(_Needle_length_el); - // zero unused elements in sequenctial permutation mask, so will be filled by 1st - const __m256i _Perm = _mm256_and_si256(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), _Mask); - _Val = _mm256_permutevar8x32_epi32(_Val, _Perm); + return _Val; + } else { + static_assert(false, "Unexpected amount"); } - - return _Val; - } else { - static_assert(false, "Unexpected amount"); } - } - template - static __m256i _Shuffle_avx(const __m256i _Val) noexcept { - if constexpr (_Amount == 1) { - return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(2, 3, 0, 1)); - } else if constexpr (_Amount == 2) { - return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); - } else if constexpr (_Amount == 4) { - return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); - } else { - static_assert(false, "Unexpected amount"); + template + static __m256i _Shuffle_avx(const __m256i _Val) noexcept { + if constexpr (_Amount == 1) { + return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else if constexpr (_Amount == 2) { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } else { + static_assert(false, "Unexpected amount"); + } } - } - }; - - struct _Find_first_of_traits_8 : _Find_traits_8 { - using _Ty = uint64_t; - - template - static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { - if constexpr (_Amount == 1) { - return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); - } else if constexpr (_Amount == 2) { - return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); - } else if constexpr (_Amount == 4) { - if (_Needle_length_el < 4) { - return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(0, 2, 1, 0)); + }; + + template + const __m256i __shuffle_step(const __m256i _Data1, const __m256i _Data2s0) { + __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); + if constexpr (_Needle_length_el_magnitude >= 2) { + const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s1)); + if constexpr (_Needle_length_el_magnitude >= 4) { + const __m256i _Data2s2 = _Traits::_Shuffle_avx<2>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s2)); + const __m256i _Data2s3 = _Traits::_Shuffle_avx<1>(_Data2s2); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s3)); + if constexpr (_Needle_length_el_magnitude >= 8) { + const __m256i _Data2s4 = _Traits::_Shuffle_avx<4>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s4)); + const __m256i _Data2s5 = _Traits::_Shuffle_avx<1>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s5)); + const __m256i _Data2s6 = _Traits::_Shuffle_avx<2>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s6)); + const __m256i _Data2s7 = _Traits::_Shuffle_avx<1>(_Data2s6); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s7)); + } } - - return _Val; - } else { - static_assert(false, "Unexpected amount"); } + return _Eq; } - template - static __m256i _Shuffle_avx(const __m256i _Val) noexcept { - if constexpr (_Amount == 1) { - return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); - } else if constexpr (_Amount == 2) { - return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); - } else { - static_assert(false, "Unexpected amount"); - } - } - }; + template + const void* __shuffle_impl( + const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) { + using _Ty = typename _Traits::_Ty; + const __m256i _Data2 = _mm256_maskload_epi32( + reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); + const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Haystack_length & ~size_t{0x1F}); + + for (; _First1 != _Stop1; _Advance_bytes(_First1, 32)) { + const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); + const __m256i _Eq = __shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const int _Bingo = _mm256_movemask_epi8(_Eq); - template - const __m256i __std_find_first_of_trivial_shuffle_step(const __m256i _Data1, const __m256i _Data2s0) { - __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); - if constexpr (_Needle_length_el_magnitude >= 2) { - const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s1)); - if constexpr (_Needle_length_el_magnitude >= 4) { - const __m256i _Data2s2 = _Traits::_Shuffle_avx<2>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s2)); - const __m256i _Data2s3 = _Traits::_Shuffle_avx<1>(_Data2s2); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s3)); - if constexpr (_Needle_length_el_magnitude >= 8) { - const __m256i _Data2s4 = _Traits::_Shuffle_avx<4>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s4)); - const __m256i _Data2s5 = _Traits::_Shuffle_avx<1>(_Data2s4); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s5)); - const __m256i _Data2s6 = _Traits::_Shuffle_avx<2>(_Data2s4); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s6)); - const __m256i _Data2s7 = _Traits::_Shuffle_avx<1>(_Data2s6); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s7)); + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First1, _Offset); + return _First1; } } - } - return _Eq; - } - template - const void* __std_find_first_of_trivial_shuffle_impl( - const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) { - using _Ty = typename _Traits::_Ty; - const __m256i _Data2 = _mm256_maskload_epi32( - reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); - const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); - - const size_t _Haystack_length = _Byte_length(_First1, _Last1); - - const void* _Stop1 = _First1; - _Advance_bytes(_Stop1, _Haystack_length & ~size_t{0x1F}); - - for (; _First1 != _Stop1; _Advance_bytes(_First1, 32)) { - const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); - const __m256i _Eq = - __std_find_first_of_trivial_shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); - const int _Bingo = _mm256_movemask_epi8(_Eq); - - if (_Bingo != 0) { - const unsigned long _Offset = _tzcnt_u32(_Bingo); - _Advance_bytes(_First1, _Offset); - return _First1; - } - } + if (const size_t _Haystack_tail_length = _Haystack_length & 0x1C; _Haystack_tail_length != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Haystack_tail_length >> 2); + const __m256i _Data1 = _mm256_maskload_epi32(static_cast(_First1), _Tail_mask); + const __m256i _Eq = __shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); - if (const size_t _Haystack_tail_length = _Haystack_length & 0x1C; _Haystack_tail_length != 0) { - const __m256i _Tail_mask = _Avx2_tail_mask_32(_Haystack_tail_length >> 2); - const __m256i _Data1 = _mm256_maskload_epi32(static_cast(_First1), _Tail_mask); - const __m256i _Eq = - __std_find_first_of_trivial_shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); - const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First1, _Offset); + return _First1; + } - if (_Bingo != 0) { - const unsigned long _Offset = _tzcnt_u32(_Bingo); - _Advance_bytes(_First1, _Offset); - return _First1; + _Advance_bytes(_First1, _Haystack_tail_length); } - _Advance_bytes(_First1, _Haystack_tail_length); + return _First1; } - return _First1; - } - - template - const void* __stdcall __std_find_first_of_trivial_48_impl(const void* const _First1, const void* const _Last1, - const void* const _First2, const void* const _Last2) noexcept { - using _Ty = typename _Traits::_Ty; + template + const void* __stdcall __48_impl(const void* const _First1, const void* const _Last1, + const void* const _First2, const void* const _Last2) noexcept { + using _Ty = typename _Traits::_Ty; #ifndef _M_ARM64EC - if (_Use_avx2()) { - _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 + if (_Use_avx2()) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - const size_t _Needle_length = _Byte_length(_First2, _Last2); - const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); + const size_t _Needle_length = _Byte_length(_First2, _Last2); + const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); - // Special handling of small needle - // The generic approach could also handle it but with worse performance - if (_Needle_length_el == 0) { - return _Last1; - } else if (_Needle_length_el == 1) { - // This is expected to be done on an upper level with better efficeiency - return __std_find_first_of_trivial_shuffle_impl<_Traits, 1>( - _First1, _Last1, _First2, _Needle_length_el); - } else if (_Needle_length_el == 2) { - return __std_find_first_of_trivial_shuffle_impl<_Traits, 2>( - _First1, _Last1, _First2, _Needle_length_el); - } else if (_Needle_length_el <= 4) { - return __std_find_first_of_trivial_shuffle_impl<_Traits, 4>( - _First1, _Last1, _First2, _Needle_length_el); - } else if (_Needle_length_el <= 8) { - if constexpr (sizeof(_Ty) == 4) { - return __std_find_first_of_trivial_shuffle_impl<_Traits, 8>( - _First1, _Last1, _First2, _Needle_length_el); + // Special handling of small needle + // The generic approach could also handle it but with worse performance + if (_Needle_length_el == 0) { + return _Last1; + } else if (_Needle_length_el == 1) { + // This is expected to be done on an upper level with better efficeiency + return __shuffle_impl<_Traits, 1>(_First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el == 2) { + return __shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el <= 4) { + return __shuffle_impl<_Traits, 4>(_First1, _Last1, _First2, _Needle_length_el); + } else if (_Needle_length_el <= 8) { + if constexpr (sizeof(_Ty) == 4) { + return __shuffle_impl<_Traits, 8>(_First1, _Last1, _First2, _Needle_length_el); + } } - } - // Generic approach - const size_t _Needle_length_tail = _Needle_length & 0x1C; - const __m256i _Tail_mask = _Avx2_tail_mask_32(_Needle_length_tail >> 2); + // Generic approach + const size_t _Needle_length_tail = _Needle_length & 0x1C; + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Needle_length_tail >> 2); - const void* _Stop2 = _First2; - _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); + const void* _Stop2 = _First2; + _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); - for (auto _Ptr1 = static_cast(_First1); _Ptr1 != _Last1; ++_Ptr1) { - const auto _Data1 = _Traits::_Set_avx(*_Ptr1); - for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { - const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); - const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Eq)) { - return _Ptr1; + for (auto _Ptr1 = static_cast(_First1); _Ptr1 != _Last1; ++_Ptr1) { + const auto _Data1 = _Traits::_Set_avx(*_Ptr1); + for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { + const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); + const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); + if (!_mm256_testz_si256(_Eq, _Eq)) { + return _Ptr1; + } } - } - if (_Needle_length_tail != 0) { - const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); - const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Tail_mask)) { - return _Ptr1; + if (_Needle_length_tail != 0) { + const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); + const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); + if (!_mm256_testz_si256(_Eq, _Tail_mask)) { + return _Ptr1; + } } } - } - return _Last1; - } + return _Last1; + } #endif // !_M_ARM64EC - return __std_find_first_of_trivial_fallback<_Ty>(_First1, _Last1, _First2, _Last2); - } - + return __fallback<_Ty>(_First1, _Last1, _First2, _Last2); + } + } // namespace __std_find_first_of template __declspec(noalias) size_t __stdcall __std_mismatch_impl( @@ -2568,22 +2563,22 @@ __declspec(noalias) size_t const void* __stdcall __std_find_first_of_trivial_1( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_pcmpestri_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__pcmpestri_impl(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_2( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_pcmpestri_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__pcmpestri_impl(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_4( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_48_impl<_Find_first_of_traits_4>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__48_impl<__std_find_first_of::_Traits_4>(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_8( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of_trivial_48_impl<_Find_first_of_traits_8>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__48_impl<__std_find_first_of::_Traits_8>(_First1, _Last1, _First2, _Last2); } __declspec(noalias) size_t From 810c695304546be32adff1a2ad5f2b1784e3f728 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 10:18:35 +0300 Subject: [PATCH 06/23] Swap _Tmp1 and _Tmp2 --- stl/src/vector_algorithms.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b03840d97b5..85774b4cc36 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2075,9 +2075,9 @@ namespace { const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _First2, _Needle_length); - const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp1)); + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First2, _Needle_length); + const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp2)); const size_t _Haystack_length = _Byte_length(_First1, _Last1); const void* _Stop_at = _First1; @@ -2097,9 +2097,9 @@ namespace { const size_t _Last_part_size = _Haystack_length & 0xF; const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp2)); + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First1, _Last_part_size); + const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); if (_mm_cmpestrc(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op)) { const int _Pos = _mm_cmpestri(_Data2, _Needle_length_el, _Data1, _Last_part_size_el, _Op); From df433ba706c6e03610577ff45850057b11dd8a1c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 10:19:08 +0300 Subject: [PATCH 07/23] Format --- stl/src/vector_algorithms.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 85774b4cc36..7418c5900c1 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2060,8 +2060,8 @@ namespace { } template - const void* __stdcall __pcmpestri_impl(const void* _First1, const void* const _Last1, - const void* const _First2, const void* const _Last2) noexcept { + const void* __stdcall __pcmpestri_impl(const void* _First1, const void* const _Last1, const void* const _First2, + const void* const _Last2) noexcept { #ifndef _M_ARM64EC if (_Use_sse42()) { constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY @@ -2344,8 +2344,8 @@ namespace { } template - const void* __stdcall __48_impl(const void* const _First1, const void* const _Last1, - const void* const _First2, const void* const _Last2) noexcept { + const void* __stdcall __48_impl(const void* const _First1, const void* const _Last1, const void* const _First2, + const void* const _Last2) noexcept { using _Ty = typename _Traits::_Ty; #ifndef _M_ARM64EC if (_Use_avx2()) { From c5e6c9e5723d8a32215838dc9100e5b4a3e6acdc Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 10:20:46 +0300 Subject: [PATCH 08/23] Swap other _Tmp1 and _Tmp2 too --- stl/src/vector_algorithms.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 7418c5900c1..cc36cdcf576 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2115,9 +2115,9 @@ namespace { const int _Last_needle_length = static_cast(_Needle_length & 0xF); - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _Last_needle, _Last_needle_length); - const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _Last_needle, _Last_needle_length); + const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp2)); const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); constexpr int _Not_found = 16; // arbitrary value greater than any found value @@ -2163,9 +2163,9 @@ namespace { const size_t _Last_part_size = _Haystack_length & 0xF; const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp2)); + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First1, _Last_part_size); + const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); _Found_pos = _Last_part_size_el; From 67815767fe523950a80401123f1181c0eaf7ffd1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 12:47:36 +0300 Subject: [PATCH 09/23] -newline --- stl/inc/algorithm | 1 - 1 file changed, 1 deletion(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index a9c776a08e3..ec16e770040 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -67,7 +67,6 @@ const void* __stdcall __std_find_first_of_trivial_4( const void* __stdcall __std_find_first_of_trivial_8( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; - __declspec(noalias) _Min_max_1i __stdcall __std_minmax_1i(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_1u __stdcall __std_minmax_1u(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_2i __stdcall __std_minmax_2i(const void* _First, const void* _Last) noexcept; From 42408744fc7c2a62096a951cced4847ee1c5bf66 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 17:08:05 +0300 Subject: [PATCH 10/23] Don't have _Needle_length_el == 1 code path --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cc36cdcf576..96fa99cf2c6 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2359,8 +2359,8 @@ namespace { if (_Needle_length_el == 0) { return _Last1; } else if (_Needle_length_el == 1) { - // This is expected to be done on an upper level with better efficeiency - return __shuffle_impl<_Traits, 1>(_First1, _Last1, _First2, _Needle_length_el); + // This is expected to be forwarded to 'find' on an upper level + _CSTD abort(); } else if (_Needle_length_el == 2) { return __shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); } else if (_Needle_length_el <= 4) { From 5122201e8a22f6afc40c0001793d5ec7bd5867a5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 17:09:58 +0300 Subject: [PATCH 11/23] spelling --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 96fa99cf2c6..917a5b2e770 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2217,7 +2217,7 @@ namespace { } else if constexpr (_Amount == 8) { if (_Needle_length_el < _Amount) { const __m256i _Mask = _Avx2_tail_mask_32(_Needle_length_el); - // zero unused elements in sequenctial permutation mask, so will be filled by 1st + // zero unused elements in sequential permutation mask, so will be filled by 1st const __m256i _Perm = _mm256_and_si256(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), _Mask); _Val = _mm256_permutevar8x32_epi32(_Val, _Perm); } From 0028fa37588ea6ab5ca1e21ea28279360c2418d5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 17:40:56 +0300 Subject: [PATCH 12/23] missing include --- stl/src/vector_algorithms.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 917a5b2e770..6e566e58fb3 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -8,6 +8,7 @@ #if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC #include <__msvc_minmax.hpp> #include +#include #include #include From 710a7b4a67bcdfa79eaa002d0e3947f3778b5459 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 14 Apr 2024 21:35:35 +0300 Subject: [PATCH 13/23] unreachable --- stl/src/vector_algorithms.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 6e566e58fb3..acfff6d04ba 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -8,7 +8,6 @@ #if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC #include <__msvc_minmax.hpp> #include -#include #include #include @@ -2360,8 +2359,7 @@ namespace { if (_Needle_length_el == 0) { return _Last1; } else if (_Needle_length_el == 1) { - // This is expected to be forwarded to 'find' on an upper level - _CSTD abort(); + _STL_UNREACHABLE; // This is expected to be forwarded to 'find' on an upper level } else if (_Needle_length_el == 2) { return __shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); } else if (_Needle_length_el <= 4) { From 09a29734e85fb3e6e0d15a6c9a2beabc6f91542d Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 14 Apr 2024 16:14:14 -0700 Subject: [PATCH 14/23] Drop unnecessary `typename` when `using`. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index acfff6d04ba..4924997928a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2303,7 +2303,7 @@ namespace { template const void* __shuffle_impl( const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) { - using _Ty = typename _Traits::_Ty; + using _Ty = _Traits::_Ty; const __m256i _Data2 = _mm256_maskload_epi32( reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); @@ -2346,7 +2346,7 @@ namespace { template const void* __stdcall __48_impl(const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { - using _Ty = typename _Traits::_Ty; + using _Ty = _Traits::_Ty; #ifndef _M_ARM64EC if (_Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 From 39bcff85b40577014af5686daf7a43d7c84038b5 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 14 Apr 2024 16:19:16 -0700 Subject: [PATCH 15/23] Add `noexcept`. --- stl/src/vector_algorithms.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 4924997928a..a2e8d4f25bd 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2041,8 +2041,8 @@ namespace { namespace __std_find_first_of { template - const void* __stdcall __fallback( - const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) { + const void* __stdcall __fallback(const void* _First1, const void* const _Last1, const void* const _First2, + const void* const _Last2) noexcept { auto _Ptr_haystack = static_cast(_First1); const auto _Ptr_haystack_end = static_cast(_Last1); const auto _Ptr_needle = static_cast(_First2); @@ -2275,7 +2275,7 @@ namespace { }; template - const __m256i __shuffle_step(const __m256i _Data1, const __m256i _Data2s0) { + const __m256i __shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); if constexpr (_Needle_length_el_magnitude >= 2) { const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); @@ -2301,8 +2301,8 @@ namespace { } template - const void* __shuffle_impl( - const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) { + const void* __shuffle_impl(const void* _First1, const void* const _Last1, const void* const _First2, + const size_t _Needle_length_el) noexcept { using _Ty = _Traits::_Ty; const __m256i _Data2 = _mm256_maskload_epi32( reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); From 42a56755faf1a918805e8b07308ae17e48087338 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 14 Apr 2024 16:21:28 -0700 Subject: [PATCH 16/23] `__48_impl` => `__4_8_impl` --- stl/src/vector_algorithms.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index a2e8d4f25bd..b8f27c36f97 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2344,7 +2344,7 @@ namespace { } template - const void* __stdcall __48_impl(const void* const _First1, const void* const _Last1, const void* const _First2, + const void* __stdcall __4_8_impl(const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { using _Ty = _Traits::_Ty; #ifndef _M_ARM64EC @@ -2572,12 +2572,12 @@ const void* __stdcall __std_find_first_of_trivial_2( const void* __stdcall __std_find_first_of_trivial_4( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__48_impl<__std_find_first_of::_Traits_4>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__4_8_impl<__std_find_first_of::_Traits_4>(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_8( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__48_impl<__std_find_first_of::_Traits_8>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::__4_8_impl<__std_find_first_of::_Traits_8>(_First1, _Last1, _First2, _Last2); } __declspec(noalias) size_t From a528021570e9c7ad8afabfa279447651f1fe6db4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 15 Apr 2024 07:34:20 +0300 Subject: [PATCH 17/23] more ARM64EC guards --- stl/src/vector_algorithms.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b8f27c36f97..78ab54ebc8d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2201,7 +2201,7 @@ namespace { struct _Traits_4 : _Find_traits_4 { using _Ty = uint32_t; - +#ifndef _M_ARM64EC template static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { if constexpr (_Amount == 1) { @@ -2240,11 +2240,12 @@ namespace { static_assert(false, "Unexpected amount"); } } +#endif // !_M_ARM64EC }; struct _Traits_8 : _Find_traits_8 { using _Ty = uint64_t; - +#ifndef _M_ARM64EC template static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { if constexpr (_Amount == 1) { @@ -2272,8 +2273,10 @@ namespace { static_assert(false, "Unexpected amount"); } } +#endif // !_M_ARM64EC }; +#ifndef _M_ARM64EC template const __m256i __shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); @@ -2343,6 +2346,8 @@ namespace { return _First1; } +#endif // !_M_ARM64EC + template const void* __stdcall __4_8_impl(const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { From 1d71a47b5eb413480b581bd10ad236caf2e41792 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 13:24:05 -0700 Subject: [PATCH 18/23] Use uppercase `_Ugly` names. `__fallback` => `_Fallback` `__shuffle_step` => `_Shuffle_step` `__shuffle_impl` => `_Shuffle_impl` `__pcmpestri_impl` => `_Impl_pcmpestri` `__4_8_impl` => `_Impl_4_8` --- stl/src/vector_algorithms.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 78ab54ebc8d..22ad5b687b0 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2041,7 +2041,7 @@ namespace { namespace __std_find_first_of { template - const void* __stdcall __fallback(const void* _First1, const void* const _Last1, const void* const _First2, + const void* __stdcall _Fallback(const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { auto _Ptr_haystack = static_cast(_First1); const auto _Ptr_haystack_end = static_cast(_Last1); @@ -2060,7 +2060,7 @@ namespace { } template - const void* __stdcall __pcmpestri_impl(const void* _First1, const void* const _Last1, const void* const _First2, + const void* __stdcall _Impl_pcmpestri(const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC if (_Use_sse42()) { @@ -2196,7 +2196,7 @@ namespace { } } #endif // !_M_ARM64EC - return __fallback<_Ty>(_First1, _Last1, _First2, _Last2); + return _Fallback<_Ty>(_First1, _Last1, _First2, _Last2); } struct _Traits_4 : _Find_traits_4 { @@ -2278,7 +2278,7 @@ namespace { #ifndef _M_ARM64EC template - const __m256i __shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { + const __m256i _Shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); if constexpr (_Needle_length_el_magnitude >= 2) { const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); @@ -2304,7 +2304,7 @@ namespace { } template - const void* __shuffle_impl(const void* _First1, const void* const _Last1, const void* const _First2, + const void* _Shuffle_impl(const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Needle_length_el) noexcept { using _Ty = _Traits::_Ty; const __m256i _Data2 = _mm256_maskload_epi32( @@ -2318,7 +2318,7 @@ namespace { for (; _First1 != _Stop1; _Advance_bytes(_First1, 32)) { const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); - const __m256i _Eq = __shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const __m256i _Eq = _Shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); const int _Bingo = _mm256_movemask_epi8(_Eq); if (_Bingo != 0) { @@ -2331,7 +2331,7 @@ namespace { if (const size_t _Haystack_tail_length = _Haystack_length & 0x1C; _Haystack_tail_length != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Haystack_tail_length >> 2); const __m256i _Data1 = _mm256_maskload_epi32(static_cast(_First1), _Tail_mask); - const __m256i _Eq = __shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); + const __m256i _Eq = _Shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); if (_Bingo != 0) { @@ -2349,7 +2349,7 @@ namespace { #endif // !_M_ARM64EC template - const void* __stdcall __4_8_impl(const void* const _First1, const void* const _Last1, const void* const _First2, + const void* __stdcall _Impl_4_8(const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { using _Ty = _Traits::_Ty; #ifndef _M_ARM64EC @@ -2366,12 +2366,12 @@ namespace { } else if (_Needle_length_el == 1) { _STL_UNREACHABLE; // This is expected to be forwarded to 'find' on an upper level } else if (_Needle_length_el == 2) { - return __shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); + return _Shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); } else if (_Needle_length_el <= 4) { - return __shuffle_impl<_Traits, 4>(_First1, _Last1, _First2, _Needle_length_el); + return _Shuffle_impl<_Traits, 4>(_First1, _Last1, _First2, _Needle_length_el); } else if (_Needle_length_el <= 8) { if constexpr (sizeof(_Ty) == 4) { - return __shuffle_impl<_Traits, 8>(_First1, _Last1, _First2, _Needle_length_el); + return _Shuffle_impl<_Traits, 8>(_First1, _Last1, _First2, _Needle_length_el); } } @@ -2404,7 +2404,7 @@ namespace { return _Last1; } #endif // !_M_ARM64EC - return __fallback<_Ty>(_First1, _Last1, _First2, _Last2); + return _Fallback<_Ty>(_First1, _Last1, _First2, _Last2); } } // namespace __std_find_first_of @@ -2567,22 +2567,22 @@ __declspec(noalias) size_t const void* __stdcall __std_find_first_of_trivial_1( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__pcmpestri_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::_Impl_pcmpestri(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_2( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__pcmpestri_impl(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::_Impl_pcmpestri(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_4( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__4_8_impl<__std_find_first_of::_Traits_4>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::_Impl_4_8<__std_find_first_of::_Traits_4>(_First1, _Last1, _First2, _Last2); } const void* __stdcall __std_find_first_of_trivial_8( const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { - return __std_find_first_of::__4_8_impl<__std_find_first_of::_Traits_8>(_First1, _Last1, _First2, _Last2); + return __std_find_first_of::_Impl_4_8<__std_find_first_of::_Traits_8>(_First1, _Last1, _First2, _Last2); } __declspec(noalias) size_t From cbd0d6e8658be118d78e5244423829d2301eb700 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 14:45:42 -0700 Subject: [PATCH 19/23] After checking `_Amount == 8`, directly say `8`. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 22ad5b687b0..b72095ec6bc 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2215,7 +2215,7 @@ namespace { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); } else if constexpr (_Amount == 8) { - if (_Needle_length_el < _Amount) { + if (_Needle_length_el < 8) { const __m256i _Mask = _Avx2_tail_mask_32(_Needle_length_el); // zero unused elements in sequential permutation mask, so will be filled by 1st const __m256i _Perm = _mm256_and_si256(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), _Mask); From 6cb45cb6a600e56e90e0f6362091199ade223e8b Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 14:48:18 -0700 Subject: [PATCH 20/23] Mark `_Val` as `const`. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b72095ec6bc..333454e41c8 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2247,7 +2247,7 @@ namespace { using _Ty = uint64_t; #ifndef _M_ARM64EC template - static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { + static __m256i _Spread_avx(const __m256i _Val, const size_t _Needle_length_el) noexcept { if constexpr (_Amount == 1) { return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); } else if constexpr (_Amount == 2) { From b447a9bcfd07090bbab69afcd816d6f495ed1e2a Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 14:50:47 -0700 Subject: [PATCH 21/23] Remove `const` from `__m256i` return type. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 333454e41c8..b8436fa8406 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2278,7 +2278,7 @@ namespace { #ifndef _M_ARM64EC template - const __m256i _Shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { + __m256i _Shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); if constexpr (_Needle_length_el_magnitude >= 2) { const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); From 1cae60f7cc1f780e7a431f62be144f7d28ca8a28 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 15:09:15 -0700 Subject: [PATCH 22/23] `!_mm256_testz_si256(ARGS)` => `_mm256_testz_si256(ARGS) == 0` --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b8436fa8406..be1746c07b4 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2387,7 +2387,7 @@ namespace { for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Eq)) { + if (_mm256_testz_si256(_Eq, _Eq) == 0) { return _Ptr1; } } @@ -2395,7 +2395,7 @@ namespace { if (_Needle_length_tail != 0) { const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Tail_mask)) { + if (_mm256_testz_si256(_Eq, _Tail_mask) == 0) { return _Ptr1; } } From 74990a687610fa29ec6925516ea2fb723cf23d5f Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 15 Apr 2024 15:42:53 -0700 Subject: [PATCH 23/23] Revert "`!_mm256_testz_si256(ARGS)` => `_mm256_testz_si256(ARGS) == 0`" This reverts commit 1cae60f7cc1f780e7a431f62be144f7d28ca8a28. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index be1746c07b4..b8436fa8406 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2387,7 +2387,7 @@ namespace { for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (_mm256_testz_si256(_Eq, _Eq) == 0) { + if (!_mm256_testz_si256(_Eq, _Eq)) { return _Ptr1; } } @@ -2395,7 +2395,7 @@ namespace { if (_Needle_length_tail != 0) { const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (_mm256_testz_si256(_Eq, _Tail_mask) == 0) { + if (!_mm256_testz_si256(_Eq, _Tail_mask)) { return _Ptr1; } }