From 1523b9012bbaa928fcbaff77ce090b38e74d1d83 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 23 Apr 2024 23:11:11 +0300 Subject: [PATCH 1/5] `find_first_of` vectorized: generalize fast approach for 4 and 8 byte elements --- stl/src/vector_algorithms.cpp | 166 ++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 77 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 6383f415e95..4f72f17654b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2204,7 +2204,9 @@ namespace { #ifndef _M_ARM64EC template static __m256i _Spread_avx(__m256i _Val, const size_t _Needle_length_el) noexcept { - if constexpr (_Amount == 1) { + if constexpr (_Amount == 0) { + return _mm256_undefined_si256(); + } else if constexpr (_Amount == 1) { return _mm256_broadcastd_epi32(_mm256_castsi256_si128(_Val)); } else if constexpr (_Amount == 2) { return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); @@ -2248,7 +2250,9 @@ namespace { #ifndef _M_ARM64EC template static __m256i _Spread_avx(const __m256i _Val, const size_t _Needle_length_el) noexcept { - if constexpr (_Amount == 1) { + if constexpr (_Amount == 0) { + return _mm256_undefined_si256(); + } else if constexpr (_Amount == 1) { return _mm256_broadcastq_epi64(_mm256_castsi256_si128(_Val)); } else if constexpr (_Amount == 2) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 1, 0)); @@ -2265,7 +2269,9 @@ namespace { template static __m256i _Shuffle_avx(const __m256i _Val) noexcept { - if constexpr (_Amount == 1) { + if constexpr (_Amount == 0) { + return _mm256_undefined_si256(); + } else if constexpr (_Amount == 1) { return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); } else if constexpr (_Amount == 2) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); @@ -2279,37 +2285,42 @@ namespace { #ifndef _M_ARM64EC template __m256i _Shuffle_step(const __m256i _Data1, const __m256i _Data2s0) noexcept { - __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); - if constexpr (_Needle_length_el_magnitude >= 2) { - const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s1)); - if constexpr (_Needle_length_el_magnitude >= 4) { - const __m256i _Data2s2 = _Traits::_Shuffle_avx<2>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s2)); - const __m256i _Data2s3 = _Traits::_Shuffle_avx<1>(_Data2s2); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s3)); - if constexpr (_Needle_length_el_magnitude >= 8) { - const __m256i _Data2s4 = _Traits::_Shuffle_avx<4>(_Data2s0); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s4)); - const __m256i _Data2s5 = _Traits::_Shuffle_avx<1>(_Data2s4); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s5)); - const __m256i _Data2s6 = _Traits::_Shuffle_avx<2>(_Data2s4); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s6)); - const __m256i _Data2s7 = _Traits::_Shuffle_avx<1>(_Data2s6); - _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s7)); + __m256i _Eq = _mm256_setzero_si256(); + if constexpr (_Needle_length_el_magnitude >= 1) { + _Eq = _Traits::_Cmp_avx(_Data1, _Data2s0); + if constexpr (_Needle_length_el_magnitude >= 2) { + const __m256i _Data2s1 = _Traits::_Shuffle_avx<1>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s1)); + if constexpr (_Needle_length_el_magnitude >= 4) { + const __m256i _Data2s2 = _Traits::_Shuffle_avx<2>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s2)); + const __m256i _Data2s3 = _Traits::_Shuffle_avx<1>(_Data2s2); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s3)); + if constexpr (_Needle_length_el_magnitude >= 8) { + const __m256i _Data2s4 = _Traits::_Shuffle_avx<4>(_Data2s0); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s4)); + const __m256i _Data2s5 = _Traits::_Shuffle_avx<1>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s5)); + const __m256i _Data2s6 = _Traits::_Shuffle_avx<2>(_Data2s4); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s6)); + const __m256i _Data2s7 = _Traits::_Shuffle_avx<1>(_Data2s6); + _Eq = _mm256_or_si256(_Eq, _Traits::_Cmp_avx(_Data1, _Data2s7)); + } } } } return _Eq; } - template + template const void* _Shuffle_impl(const void* _First1, const void* const _Last1, const void* const _First2, - const size_t _Needle_length_el) noexcept { - using _Ty = _Traits::_Ty; - const __m256i _Data2 = _mm256_maskload_epi32( - reinterpret_cast(_First2), _Avx2_tail_mask_32(_Needle_length_el * (sizeof(_Ty) / 4))); - const __m256i _Data2s0 = _Traits::_Spread_avx<_Needle_length_el_magnitude>(_Data2, _Needle_length_el); + const void* const _Stop2, const size_t _Last2_length_el) noexcept { + using _Ty = _Traits::_Ty; + constexpr size_t _Length_el = 32 / sizeof(_Ty); + + const __m256i _Last2val = _mm256_maskload_epi32( + reinterpret_cast(_Stop2), _Avx2_tail_mask_32(_Last2_length_el * (sizeof(_Ty) / 4))); + const __m256i _Last2s0 = _Traits::_Spread_avx<_Last2_length_el_magnitude>(_Last2val, _Last2_length_el); const size_t _Haystack_length = _Byte_length(_First1, _Last1); @@ -2318,10 +2329,16 @@ namespace { for (; _First1 != _Stop1; _Advance_bytes(_First1, 32)) { const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); - const __m256i _Eq = _Shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); - const int _Bingo = _mm256_movemask_epi8(_Eq); + __m256i _Eq = _Shuffle_step<_Traits, _Last2_length_el_magnitude>(_Data1, _Last2s0); - if (_Bingo != 0) { + if constexpr (_Large) { + for (const void* _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { + const __m256i _Data2s0 = _mm256_loadu_si256(static_cast(_Ptr2)); + _Eq = _mm256_or_si256(_Eq, _Shuffle_step<_Traits, _Length_el>(_Data1, _Data2s0)); + } + } + + if (const int _Bingo = _mm256_movemask_epi8(_Eq); _Bingo != 0) { const unsigned long _Offset = _tzcnt_u32(_Bingo); _Advance_bytes(_First1, _Offset); return _First1; @@ -2331,10 +2348,16 @@ namespace { if (const size_t _Haystack_tail_length = _Haystack_length & 0x1C; _Haystack_tail_length != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Haystack_tail_length >> 2); const __m256i _Data1 = _mm256_maskload_epi32(static_cast(_First1), _Tail_mask); - const __m256i _Eq = _Shuffle_step<_Traits, _Needle_length_el_magnitude>(_Data1, _Data2s0); - const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); + __m256i _Eq = _Shuffle_step<_Traits, _Last2_length_el_magnitude>(_Data1, _Last2s0); - if (_Bingo != 0) { + if constexpr (_Large) { + for (const void* _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { + const __m256i _Data2s0 = _mm256_loadu_si256(static_cast(_Ptr2)); + _Eq = _mm256_or_si256(_Eq, _Shuffle_step<_Traits, _Length_el>(_Data1, _Data2s0)); + } + } + + if (const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Eq, _Tail_mask)); _Bingo != 0) { const unsigned long _Offset = _tzcnt_u32(_Bingo); _Advance_bytes(_First1, _Offset); return _First1; @@ -2346,6 +2369,28 @@ namespace { return _First1; } + template + const void* _Shuffle_impl_dispatch_magnitude(const void* _First1, const void* const _Last1, + const void* const _First2, const void* const _Stop2, const size_t _Last2_length_el) noexcept { + if (_Last2_length_el == 0) { + return _Shuffle_impl<_Traits, _Large, 0>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); + } else if (_Last2_length_el == 1) { + return _Shuffle_impl<_Traits, _Large, 1>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); + } else if (_Last2_length_el == 2) { + return _Shuffle_impl<_Traits, _Large, 2>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); + } else if (_Last2_length_el <= 4) { + return _Shuffle_impl<_Traits, _Large, 4>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); + } else if (_Last2_length_el <= 8) { + if constexpr (sizeof(_Traits::_Ty) == 4) { + return _Shuffle_impl<_Traits, _Large, 8>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); + } else { + _STL_UNREACHABLE; + } + } else { + _STL_UNREACHABLE; + } + } + #endif // !_M_ARM64EC template @@ -2356,52 +2401,19 @@ namespace { if (_Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - const size_t _Needle_length = _Byte_length(_First2, _Last2); - const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); - - // Special handling of small needle - // The generic approach could also handle it but with worse performance - if (_Needle_length_el == 0) { - return _Last1; - } else if (_Needle_length_el == 1) { - _STL_UNREACHABLE; // This is expected to be forwarded to 'find' on an upper level - } else if (_Needle_length_el == 2) { - return _Shuffle_impl<_Traits, 2>(_First1, _Last1, _First2, _Needle_length_el); - } else if (_Needle_length_el <= 4) { - return _Shuffle_impl<_Traits, 4>(_First1, _Last1, _First2, _Needle_length_el); - } else if (_Needle_length_el <= 8) { - if constexpr (sizeof(_Ty) == 4) { - return _Shuffle_impl<_Traits, 8>(_First1, _Last1, _First2, _Needle_length_el); - } - } - - // Generic approach - const size_t _Needle_length_tail = _Needle_length & 0x1C; - const __m256i _Tail_mask = _Avx2_tail_mask_32(_Needle_length_tail >> 2); + const size_t _Needle_length = _Byte_length(_First2, _Last2); + const size_t _Last_needle_length = _Needle_length & 0x1F; + const size_t _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); - const void* _Stop2 = _First2; - _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); - - for (auto _Ptr1 = static_cast(_First1); _Ptr1 != _Last1; ++_Ptr1) { - const auto _Data1 = _Traits::_Set_avx(*_Ptr1); - for (auto _Ptr2 = _First2; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 32)) { - const __m256i _Data2 = _mm256_loadu_si256(static_cast(_Ptr2)); - const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Eq)) { - return _Ptr1; - } - } - - if (_Needle_length_tail != 0) { - const __m256i _Data2 = _mm256_maskload_epi32(static_cast(_Stop2), _Tail_mask); - const __m256i _Eq = _Traits::_Cmp_avx(_Data1, _Data2); - if (!_mm256_testz_si256(_Eq, _Tail_mask)) { - return _Ptr1; - } - } + if (size_t _Needle_length_large = _Needle_length & ~size_t{0x1F}; _Needle_length_large != 0) { + const void* _Stop2 = _First2; + _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); + return _Shuffle_impl_dispatch_magnitude<_Traits, true>( + _First1, _Last1, _First2, _Stop2, _Last_needle_length_el); + } else { + return _Shuffle_impl_dispatch_magnitude<_Traits, false>( + _First1, _Last1, _First2, _First2, _Last_needle_length_el); } - - return _Last1; } #endif // !_M_ARM64EC return _Fallback<_Ty>(_First1, _Last1, _First2, _Last2); From 8efb7e0cb1cb2ea487645dc11fed0b2abdd9d49e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 23 Apr 2024 23:29:39 +0300 Subject: [PATCH 2/5] unnecessary branch --- stl/src/vector_algorithms.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 4f72f17654b..17fba3d2cce 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2269,9 +2269,7 @@ namespace { template static __m256i _Shuffle_avx(const __m256i _Val) noexcept { - if constexpr (_Amount == 0) { - return _mm256_undefined_si256(); - } else if constexpr (_Amount == 1) { + if constexpr (_Amount == 1) { return _mm256_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); } else if constexpr (_Amount == 2) { return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); From eaa5f7fb0c4cecd13fe455795e4019728dcb48c4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 23 Apr 2024 23:43:27 +0300 Subject: [PATCH 3/5] More concise unreachable --- stl/src/vector_algorithms.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 17fba3d2cce..5d968a3253a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2381,12 +2381,10 @@ namespace { } else if (_Last2_length_el <= 8) { if constexpr (sizeof(_Traits::_Ty) == 4) { return _Shuffle_impl<_Traits, _Large, 8>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); - } else { - _STL_UNREACHABLE; } - } else { - _STL_UNREACHABLE; } + + _STL_UNREACHABLE; } #endif // !_M_ARM64EC From 6f3a3d23e13e38133c25048e73314aa3d9e1eacd Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 24 Apr 2024 13:49:45 -0700 Subject: [PATCH 4/5] Add `const`. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 5d968a3253a..ed3fff362ec 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2368,7 +2368,7 @@ namespace { } template - const void* _Shuffle_impl_dispatch_magnitude(const void* _First1, const void* const _Last1, + const void* _Shuffle_impl_dispatch_magnitude(const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Stop2, const size_t _Last2_length_el) noexcept { if (_Last2_length_el == 0) { return _Shuffle_impl<_Traits, _Large, 0>(_First1, _Last1, _First2, _Stop2, _Last2_length_el); @@ -2401,7 +2401,7 @@ namespace { const size_t _Last_needle_length = _Needle_length & 0x1F; const size_t _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); - if (size_t _Needle_length_large = _Needle_length & ~size_t{0x1F}; _Needle_length_large != 0) { + if (const size_t _Needle_length_large = _Needle_length & ~size_t{0x1F}; _Needle_length_large != 0) { const void* _Stop2 = _First2; _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); return _Shuffle_impl_dispatch_magnitude<_Traits, true>( From ef4e2a53f4c88e2eb0e49f17bfac7c6e62c911d6 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 24 Apr 2024 13:50:02 -0700 Subject: [PATCH 5/5] We've extracted `_Needle_length_large`. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ed3fff362ec..bf5ebe09908 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2403,7 +2403,7 @@ namespace { if (const size_t _Needle_length_large = _Needle_length & ~size_t{0x1F}; _Needle_length_large != 0) { const void* _Stop2 = _First2; - _Advance_bytes(_Stop2, _Needle_length & ~size_t{0x1F}); + _Advance_bytes(_Stop2, _Needle_length_large); return _Shuffle_impl_dispatch_magnitude<_Traits, true>( _First1, _Last1, _First2, _Stop2, _Last_needle_length_el); } else {