From d98bfa0446bcf2480fd81d12e775a90098cf39ca Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 10 Apr 2024 15:04:44 +0300 Subject: [PATCH 1/4] Don't go SSE path if we're on AVX path --- benchmarks/src/find_and_count.cpp | 48 ++++++++++------- stl/src/vector_algorithms.cpp | 86 ++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 42 deletions(-) diff --git a/benchmarks/src/find_and_count.cpp b/benchmarks/src/find_and_count.cpp index 7b205aee6a0..598777a33fc 100644 --- a/benchmarks/src/find_and_count.cpp +++ b/benchmarks/src/find_and_count.cpp @@ -6,6 +6,7 @@ #include #include #include +#include enum class Op { FindSized, @@ -15,39 +16,50 @@ enum class Op { using namespace std; -template +template void bm(benchmark::State& state) { - T a[Size]; + const auto size = static_cast(state.range(0)); + const auto pos = static_cast(state.range(1)); - fill_n(a, Size, T{'0'}); - if constexpr (Pos < Size) { - a[Pos] = T{'1'}; + vector a(size, T{'0'}); + + if (pos < size) { + a[pos] = T{'1'}; } else { - static_assert(Operation != Op::FindUnsized); + if constexpr (Operation == Op::FindUnsized) { + abort(); + } } for (auto _ : state) { if constexpr (Operation == Op::FindSized) { - benchmark::DoNotOptimize(ranges::find(a, a + Size, T{'1'})); + benchmark::DoNotOptimize(ranges::find(a.begin(), a.end(), T{'1'})); } else if constexpr (Operation == Op::FindUnsized) { - benchmark::DoNotOptimize(ranges::find(a, unreachable_sentinel, T{'1'})); + benchmark::DoNotOptimize(ranges::find(a.begin(), unreachable_sentinel, T{'1'})); } else if constexpr (Operation == Op::Count) { - benchmark::DoNotOptimize(ranges::count(a, a + Size, T{'1'})); + benchmark::DoNotOptimize(ranges::count(a.begin(), a.end(), T{'1'})); } } } -BENCHMARK(bm); -BENCHMARK(bm); -BENCHMARK(bm); +void common_args(auto bm) { + bm->Args({8021, 3056})-> + // AVX tail tests + Args({63, 62})->Args({31, 30})->Args({15, 14})->Args({7, 6}); +} + + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm); -BENCHMARK(bm); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm); -BENCHMARK(bm); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm); -BENCHMARK(bm); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK_MAIN(); diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0eb3c4cf24d..d34fd0b888a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1837,15 +1837,15 @@ namespace { template const void* __stdcall __std_find_trivial_impl(const void* _First, const void* _Last, _Ty _Val) noexcept { #ifndef _M_ARM64EC - size_t _Size_bytes = _Byte_length(_First, _Last); - - const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; - if (_Avx_size != 0 && _Use_avx2()) { + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Avx_size); + do { const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); @@ -1858,14 +1858,29 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); - _Size_bytes &= 0x1F; - } + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, _Avx_tail_size); + } - const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; - if (_Sse_size != 0 && _Use_sse42()) { + if constexpr (sizeof(_Ty) >= 4) { + return _First; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Sse_size); + do { const __m128i _Data = _mm_loadu_si128(static_cast(_First)); const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); @@ -1892,15 +1907,15 @@ namespace { const void* __stdcall __std_find_last_trivial_impl(const void* _First, const void* _Last, _Ty _Val) noexcept { const void* const _Real_last = _Last; #ifndef _M_ARM64EC - size_t _Size_bytes = _Byte_length(_First, _Last); + const size_t _Size_bytes = _Byte_length(_First, _Last); - const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; - if (_Avx_size != 0 && _Use_avx2()) { + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _Last; _Rewind_bytes(_Stop_at, _Avx_size); + do { _Rewind_bytes(_Last, 32); const __m256i _Data = _mm256_loadu_si256(static_cast(_Last)); @@ -1912,14 +1927,28 @@ namespace { return _Last; } } while (_Last != _Stop_at); - _Size_bytes &= 0x1F; - } - const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; - if (_Sse_size != 0 && _Use_sse42()) { + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + _Rewind_bytes(_Last, _Avx_tail_size); + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_Last), _Tail_mask); + const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + + if (_Bingo != 0) { + const unsigned long _Offset = _lzcnt_u32(_Bingo); + _Advance_bytes(_Last, (31 - _Offset) - (sizeof(_Ty) - 1)); + return _Last; + } + } + + if constexpr (sizeof(_Ty) >= 4) { + return _Real_last; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); const void* _Stop_at = _Last; _Rewind_bytes(_Stop_at, _Sse_size); + do { _Rewind_bytes(_Last, 16); const __m128i _Data = _mm_loadu_si128(static_cast(_Last)); @@ -1952,29 +1981,38 @@ namespace { size_t _Result = 0; #ifndef _M_ARM64EC - size_t _Size_bytes = _Byte_length(_First, _Last); + const size_t _Size_bytes = _Byte_length(_First, _Last); - const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; - if (_Avx_size != 0 && _Use_avx2()) { + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Avx_size); + do { const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); _Result += __popcnt(_Bingo); // Assume available with SSE4.2 _Advance_bytes(_First, 32); } while (_First != _Stop_at); - _Size_bytes &= 0x1F; + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + _Result += __popcnt(_Bingo); // Assume available with SSE4.2 + _Advance_bytes(_First, _Avx_tail_size); + } _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; - if (_Sse_size != 0 && _Use_sse42()) { + if constexpr (sizeof(_Ty) >= 4) { + return _Result >>= _Traits::_Shift; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Sse_size); + do { const __m128i _Data = _mm_loadu_si128(static_cast(_First)); const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); @@ -1984,8 +2022,8 @@ namespace { } #endif // !_M_ARM64EC _Result >>= _Traits::_Shift; - auto _Ptr = static_cast(_First); - for (; _Ptr != _Last; ++_Ptr) { + + for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { if (*_Ptr == _Val) { ++_Result; } From 6c8e49dfb72088a1c584c75a24bef1a2d2073a20 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 10 Apr 2024 15:55:40 +0300 Subject: [PATCH 2/4] format --- benchmarks/src/find_and_count.cpp | 6 +++--- stl/src/vector_algorithms.cpp | 13 ++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/benchmarks/src/find_and_count.cpp b/benchmarks/src/find_and_count.cpp index 598777a33fc..c1512bbd9c0 100644 --- a/benchmarks/src/find_and_count.cpp +++ b/benchmarks/src/find_and_count.cpp @@ -43,9 +43,9 @@ void bm(benchmark::State& state) { } void common_args(auto bm) { - bm->Args({8021, 3056})-> - // AVX tail tests - Args({63, 62})->Args({31, 30})->Args({15, 14})->Args({7, 6}); + bm->Args({8021, 3056}); + // AVX tail tests + bm->Args({63, 62})->Args({31, 30})->Args({15, 14})->Args({7, 6}); } diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index d34fd0b888a..46be51598df 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1838,7 +1838,7 @@ namespace { const void* __stdcall __std_find_trivial_impl(const void* _First, const void* _Last, _Ty _Val) noexcept { #ifndef _M_ARM64EC const size_t _Size_bytes = _Byte_length(_First, _Last); - + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 @@ -1858,11 +1858,12 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); - + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); - const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + const int _Bingo = + _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); if (_Bingo != 0) { const unsigned long _Offset = _tzcnt_u32(_Bingo); @@ -1932,7 +1933,8 @@ namespace { _Rewind_bytes(_Last, _Avx_tail_size); const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_Last), _Tail_mask); - const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + const int _Bingo = + _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); if (_Bingo != 0) { const unsigned long _Offset = _lzcnt_u32(_Bingo); @@ -1998,7 +2000,8 @@ namespace { if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); - const int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + const int _Bingo = + _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); _Result += __popcnt(_Bingo); // Assume available with SSE4.2 _Advance_bytes(_First, _Avx_tail_size); } From dc988b247e1b76e95c1272dc04f687ba98f2ae6b Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 11 Apr 2024 15:36:55 -0700 Subject: [PATCH 3/4] Move `_Result >>= _Traits::_Shift;` to the end of the AVX2 and SSE4.2 codepaths. --- stl/src/vector_algorithms.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 46be51598df..2349d804e3f 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2008,8 +2008,10 @@ namespace { _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + _Result >>= _Traits::_Shift; + if constexpr (sizeof(_Ty) >= 4) { - return _Result >>= _Traits::_Shift; + return _Result; } } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); @@ -2022,9 +2024,10 @@ namespace { _Result += __popcnt(_Bingo); // Assume available with SSE4.2 _Advance_bytes(_First, 16); } while (_First != _Stop_at); + + _Result >>= _Traits::_Shift; } #endif // !_M_ARM64EC - _Result >>= _Traits::_Shift; for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { if (*_Ptr == _Val) { From 8401611b7a78c14f60998ece3722779ded0d7ca3 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 11 Apr 2024 15:41:05 -0700 Subject: [PATCH 4/4] Include `` for `abort()`. --- benchmarks/src/find_and_count.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/src/find_and_count.cpp b/benchmarks/src/find_and_count.cpp index c1512bbd9c0..9c608bfe356 100644 --- a/benchmarks/src/find_and_count.cpp +++ b/benchmarks/src/find_and_count.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include