From b1aacab069e3cf3885b0925b149150603b7a2184 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 21 Apr 2024 19:13:54 +0300 Subject: [PATCH 01/27] `count` vectorization: replace `popcnt` implementation with vector counting --- stl/src/vector_algorithms.cpp | 200 +++++++++++++++++++++++++++++----- 1 file changed, 170 insertions(+), 30 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 6383f415e95..9b92afa33bb 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1734,8 +1734,6 @@ __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* const _First namespace { struct _Find_traits_1 { - static constexpr size_t _Shift = 0; - #ifndef _M_ARM64EC static __m256i _Set_avx(const uint8_t _Val) noexcept { return _mm256_set1_epi8(_Val); @@ -1752,12 +1750,41 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi8(_Lhs, _Rhs); } + + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi8(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi8(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); + const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); + const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane + const __m256i _Rx2 = _mm256_hadd_epi16(_Rx1, _mm256_setzero_si256()); // (0+1+2+3),..,(13+14+15+16),0,0,0,0 + const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend + const __m256i _Rx4 = _mm256_hadd_epi32(_Rx3, _mm256_setzero_si256()); // (0+...+7),(8+...+15),0,0 + const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+...+15),0,0,0 + return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) + + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); + const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); + const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) + const __m128i _Rx2 = _mm_hadd_epi16(_Rx1, _mm_setzero_si128()); // (0+1+2+3),..,(13+14+15+16),0,0,0,0 + const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend + const __m128i _Rx4 = _mm_hadd_epi32(_Rx3, _mm_setzero_si128()); // (0+...+7),(8+...+15),0,0 + const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+15),0,0,0 + return _mm_cvtsi128_si32(_Rx5); + } #endif // !_M_ARM64EC }; struct _Find_traits_2 { - static constexpr size_t _Shift = 1; - #ifndef _M_ARM64EC static __m256i _Set_avx(const uint16_t _Val) noexcept { return _mm256_set1_epi16(_Val); @@ -1774,12 +1801,35 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi16(_Lhs, _Rhs); } + + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi16(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi16(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Rx2 = _mm256_hadd_epi16(_Val, _mm256_setzero_si256()); // (0+1),..,(6+7),0,0,0,0 per lane + const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend + const __m256i _Rx4 = _mm256_hadd_epi32(_Rx3, _mm256_setzero_si256()); // (0+...+3),(4+...+7),0,0 + const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+...+7),0,0,0 + return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) + + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Rx2 = _mm_hadd_epi16(_Val, _mm_setzero_si128()); // (0+1),..,(6+7),0,0,0,0 + const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend + const __m128i _Rx4 = _mm_hadd_epi32(_Rx3, _mm_setzero_si128()); // (0+...+3),(4+...+7),0,0 + const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+7),0,0,0 + return _mm_cvtsi128_si32(_Rx5); + } #endif // !_M_ARM64EC }; struct _Find_traits_4 { - static constexpr size_t _Shift = 2; - #ifndef _M_ARM64EC static __m256i _Set_avx(const uint32_t _Val) noexcept { return _mm256_set1_epi32(_Val); @@ -1796,12 +1846,31 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi32(_Lhs, _Rhs); } + + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi32(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi32(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane + const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+3),0,0,0 + return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) + + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Rx4 = _mm_hadd_epi32(_Val, _mm_setzero_si128()); // (0+1),(2+3),0,0 + const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+3),0,0,0 + return _mm_cvtsi128_si32(_Rx5); + } #endif // !_M_ARM64EC }; struct _Find_traits_8 { - static constexpr size_t _Shift = 3; - #ifndef _M_ARM64EC static __m256i _Set_avx(const uint64_t _Val) noexcept { return _mm256_set1_epi64x(_Val); @@ -1818,6 +1887,34 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi64(_Lhs, _Rhs); } + + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi64(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi64(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m128i _Rx6 = _mm256_extracti128_si256(_Val, 0); + const __m128i _Rx7 = _mm256_extracti128_si256(_Val, 1); +#ifdef _M_IX86 + return _mm_cvtsi128_si32(_Rx6) + _mm_extract_epi32(_Rx6, 2) // + + _mm_cvtsi128_si32(_Rx7) + _mm_extract_epi32(_Rx7, 2); +#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv + return _mm_cvtsi128_si64(_Rx6) + _mm_extract_epi64(_Rx6, 1) // + + _mm_cvtsi128_si64(_Rx7) + _mm_extract_epi64(_Rx7, 1); +#endif // ^^^ defined(_M_X64) ^^^ + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { +#ifdef _M_IX86 + return _mm_cvtsi128_si32(_Val) + _mm_extract_epi32(_Val, 2); +#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv + return _mm_cvtsi128_si64(_Val) + _mm_extract_epi64(_Val, 1); +#endif // ^^^ defined(_M_X64) ^^^ + } #endif // !_M_ARM64EC }; @@ -1986,47 +2083,90 @@ namespace { #ifndef _M_ARM64EC const size_t _Size_bytes = _Byte_length(_First, _Last); - if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { + if (size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Avx_size); + __m256i _Count_vector = _mm256_setzero_si256(); - do { - const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); - const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); - _Result += __popcnt(_Bingo); // Assume available with SSE4.2 - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); + for (;;) { + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { + _Advance_bytes(_Stop_at, _Avx_size); + } else { + constexpr size_t _Max_portion_size = (size_t{1} << ((sizeof(_Ty) * 8) - 1)) * 32 / sizeof(_Ty); + const size_t _Portion_size = _Avx_size < _Max_portion_size ? _Avx_size : _Max_portion_size; + _Advance_bytes(_Stop_at, _Portion_size); + _Avx_size -= _Portion_size; + } + + do { + const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + _Count_vector = _Traits::_Sub_avx(_Count_vector, _Mask); + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); + + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { + break; + } else { + _Result += _Traits::_Reduce_avx(_Count_vector); + _Count_vector = _mm256_setzero_si256(); + + if (_Avx_size == 0) { + break; + } + } + } if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); - const int _Bingo = - _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); - _Result += __popcnt(_Bingo); // Assume available with SSE4.2 + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + _Count_vector = _Traits::_Sub_avx(_Count_vector, _mm256_and_si256(_Mask, _Tail_mask)); _Advance_bytes(_First, _Avx_tail_size); } - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + _Result += _Traits::_Reduce_avx(_Count_vector); - _Result >>= _Traits::_Shift; + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 if constexpr (sizeof(_Ty) >= 4) { return _Result; } - } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { + } else if (size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Sse_size); + __m128i _Count_vector = _mm_setzero_si128(); - do { - const __m128i _Data = _mm_loadu_si128(static_cast(_First)); - const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); - _Result += __popcnt(_Bingo); // Assume available with SSE4.2 - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); + for (;;) { + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { + _Advance_bytes(_Stop_at, _Sse_size); + } else { + constexpr size_t _Max_portion_size = (size_t{1} << ((sizeof(_Ty) * 8) - 1)) * 16 / sizeof(_Ty); + const size_t _Portion_size = _Sse_size < _Max_portion_size ? _Sse_size : _Max_portion_size; + _Advance_bytes(_Stop_at, _Portion_size); + _Sse_size -= _Portion_size; + } + + do { + const __m128i _Data = _mm_loadu_si128(static_cast(_First)); + const __m128i _Mask = _Traits::_Cmp_sse(_Data, _Comparand); + _Count_vector = _Traits::_Sub_sse(_Count_vector, _Mask); + _Advance_bytes(_First, 16); + } while (_First != _Stop_at); + + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { + break; + } else { + _Result += _Traits::_Reduce_sse(_Count_vector); + _Count_vector = _mm_setzero_si128(); + + if (_Sse_size == 0) { + break; + } + } + } - _Result >>= _Traits::_Shift; + _Result += _Traits::_Reduce_sse(_Count_vector); } #endif // !_M_ARM64EC From 6f134e8a9b30c94ea9ce3791113ea347a9a15210 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 21 Apr 2024 20:05:38 +0300 Subject: [PATCH 02/27] Don't do extra reduce in the end --- stl/src/vector_algorithms.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 9b92afa33bb..4d116d74084 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2108,12 +2108,12 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { break; } else { - _Result += _Traits::_Reduce_avx(_Count_vector); - _Count_vector = _mm256_setzero_si256(); - if (_Avx_size == 0) { break; } + + _Result += _Traits::_Reduce_avx(_Count_vector); + _Count_vector = _mm256_setzero_si256(); } } @@ -2157,12 +2157,12 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { break; } else { - _Result += _Traits::_Reduce_sse(_Count_vector); - _Count_vector = _mm_setzero_si128(); - if (_Sse_size == 0) { break; } + + _Result += _Traits::_Reduce_sse(_Count_vector); + _Count_vector = _mm_setzero_si128(); } } From 35d61ac7569ac9a0552503ea6c676e4bef8e368b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 21 Apr 2024 21:37:19 +0300 Subject: [PATCH 03/27] As the SSE branch has no masked tail, can reduce in a single pace --- stl/src/vector_algorithms.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 4d116d74084..711a475715f 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2154,6 +2154,8 @@ namespace { _Advance_bytes(_First, 16); } while (_First != _Stop_at); + _Result += _Traits::_Reduce_sse(_Count_vector); + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { break; } else { @@ -2161,12 +2163,9 @@ namespace { break; } - _Result += _Traits::_Reduce_sse(_Count_vector); _Count_vector = _mm_setzero_si128(); } } - - _Result += _Traits::_Reduce_sse(_Count_vector); } #endif // !_M_ARM64EC From ce8d8a50651eb6fd4fbb5eafb143d0bcb2db5c30 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 07:29:51 +0300 Subject: [PATCH 04/27] Reduce as infrequently as possible --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 711a475715f..c4799a85187 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2092,7 +2092,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Avx_size); } else { - constexpr size_t _Max_portion_size = (size_t{1} << ((sizeof(_Ty) * 8) - 1)) * 32 / sizeof(_Ty); + constexpr size_t _Max_portion_size = ((size_t{1} << (sizeof(_Ty) * 8)) - 2) * 32 / sizeof(_Ty); const size_t _Portion_size = _Avx_size < _Max_portion_size ? _Avx_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Avx_size -= _Portion_size; @@ -2141,7 +2141,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Sse_size); } else { - constexpr size_t _Max_portion_size = (size_t{1} << ((sizeof(_Ty) * 8) - 1)) * 16 / sizeof(_Ty); + constexpr size_t _Max_portion_size = ((size_t{1} << (sizeof(_Ty) * 8)) - 1) * 16 / sizeof(_Ty); const size_t _Portion_size = _Sse_size < _Max_portion_size ? _Sse_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Sse_size -= _Portion_size; From af456dbea1bdbf6870fa8a117ff208fa19733c65 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 08:30:55 +0300 Subject: [PATCH 05/27] missing range coverage --- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 9d0dc896cfd..6652e77f6d5 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -83,6 +83,10 @@ void test_case_count(const vector& input, T v) { auto expected = last_known_good_count(input.begin(), input.end(), v); auto actual = count(input.begin(), input.end(), v); assert(expected == actual); +#if _HAS_CXX20 + auto actual_r = ranges::count(input, v); + assert(actual_r == actual); +#endif // _HAS_CXX20 } template From c232f629ec02cf092c1587a9d1c9a3c617cab7c5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 08:37:01 +0300 Subject: [PATCH 06/27] test counting zeros --- .../VSO_0000000_vector_algorithms/test.cpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 6652e77f6d5..881b994474c 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -102,6 +102,17 @@ void test_count(mt19937_64& gen) { } } +template +void test_count_zero() { // text that counters don't overflow + vector input; + input.reserve(dataCount); + test_case_count(input, T{0}); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + input.push_back(0); + test_case_count(input, T{0}); + } +} + template auto last_known_good_find(FwdIt first, FwdIt last, T v) { for (; first != last; ++first) { @@ -742,6 +753,16 @@ void test_vector_algorithms(mt19937_64& gen) { test_count(gen); test_count(gen); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_count_zero(); + test_find(gen); test_find(gen); test_find(gen); From 498408cc7a5954437476c97553bd8e38a6580ee7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 08:51:20 +0300 Subject: [PATCH 07/27] compare with expected in new coverage --- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 881b994474c..d762e6b4b43 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -85,7 +85,7 @@ void test_case_count(const vector& input, T v) { assert(expected == actual); #if _HAS_CXX20 auto actual_r = ranges::count(input, v); - assert(actual_r == actual); + assert(actual_r == expected); #endif // _HAS_CXX20 } From 48efb24498385516bdf16a79001f48177cee4ec2 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 10:27:21 +0300 Subject: [PATCH 08/27] separate _Count_traits_N and reuse reduce --- stl/src/vector_algorithms.cpp | 220 +++++++++++++++++----------------- 1 file changed, 111 insertions(+), 109 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index c4799a85187..252aaeb9c27 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1750,37 +1750,6 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi8(_Lhs, _Rhs); } - - static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_sub_epi8(_Lhs, _Rhs); - } - - static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_sub_epi8(_Lhs, _Rhs); - } - - static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane - const __m256i _Rx2 = _mm256_hadd_epi16(_Rx1, _mm256_setzero_si256()); // (0+1+2+3),..,(13+14+15+16),0,0,0,0 - const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend - const __m256i _Rx4 = _mm256_hadd_epi32(_Rx3, _mm256_setzero_si256()); // (0+...+7),(8+...+15),0,0 - const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+...+15),0,0,0 - return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) - + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); - } - - static size_t _Reduce_sse(const __m128i _Val) noexcept { - const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); - const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); - const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) - const __m128i _Rx2 = _mm_hadd_epi16(_Rx1, _mm_setzero_si128()); // (0+1+2+3),..,(13+14+15+16),0,0,0,0 - const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend - const __m128i _Rx4 = _mm_hadd_epi32(_Rx3, _mm_setzero_si128()); // (0+...+7),(8+...+15),0,0 - const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+15),0,0,0 - return _mm_cvtsi128_si32(_Rx5); - } #endif // !_M_ARM64EC }; @@ -1801,31 +1770,6 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi16(_Lhs, _Rhs); } - - static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_sub_epi16(_Lhs, _Rhs); - } - - static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_sub_epi16(_Lhs, _Rhs); - } - - static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Rx2 = _mm256_hadd_epi16(_Val, _mm256_setzero_si256()); // (0+1),..,(6+7),0,0,0,0 per lane - const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend - const __m256i _Rx4 = _mm256_hadd_epi32(_Rx3, _mm256_setzero_si256()); // (0+...+3),(4+...+7),0,0 - const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+...+7),0,0,0 - return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) - + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); - } - - static size_t _Reduce_sse(const __m128i _Val) noexcept { - const __m128i _Rx2 = _mm_hadd_epi16(_Val, _mm_setzero_si128()); // (0+1),..,(6+7),0,0,0,0 - const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend - const __m128i _Rx4 = _mm_hadd_epi32(_Rx3, _mm_setzero_si128()); // (0+...+3),(4+...+7),0,0 - const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+7),0,0,0 - return _mm_cvtsi128_si32(_Rx5); - } #endif // !_M_ARM64EC }; @@ -1846,27 +1790,6 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi32(_Lhs, _Rhs); } - - static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_sub_epi32(_Lhs, _Rhs); - } - - static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_sub_epi32(_Lhs, _Rhs); - } - - static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane - const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+3),0,0,0 - return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) - + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); - } - - static size_t _Reduce_sse(const __m128i _Val) noexcept { - const __m128i _Rx4 = _mm_hadd_epi32(_Val, _mm_setzero_si128()); // (0+1),(2+3),0,0 - const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+3),0,0,0 - return _mm_cvtsi128_si32(_Rx5); - } #endif // !_M_ARM64EC }; @@ -1887,34 +1810,6 @@ namespace { static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { return _mm_cmpeq_epi64(_Lhs, _Rhs); } - - static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_sub_epi64(_Lhs, _Rhs); - } - - static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_sub_epi64(_Lhs, _Rhs); - } - - static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m128i _Rx6 = _mm256_extracti128_si256(_Val, 0); - const __m128i _Rx7 = _mm256_extracti128_si256(_Val, 1); -#ifdef _M_IX86 - return _mm_cvtsi128_si32(_Rx6) + _mm_extract_epi32(_Rx6, 2) // - + _mm_cvtsi128_si32(_Rx7) + _mm_extract_epi32(_Rx7, 2); -#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv - return _mm_cvtsi128_si64(_Rx6) + _mm_extract_epi64(_Rx6, 1) // - + _mm_cvtsi128_si64(_Rx7) + _mm_extract_epi64(_Rx7, 1); -#endif // ^^^ defined(_M_X64) ^^^ - } - - static size_t _Reduce_sse(const __m128i _Val) noexcept { -#ifdef _M_IX86 - return _mm_cvtsi128_si32(_Val) + _mm_extract_epi32(_Val, 2); -#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv - return _mm_cvtsi128_si64(_Val) + _mm_extract_epi64(_Val, 1); -#endif // ^^^ defined(_M_X64) ^^^ - } #endif // !_M_ARM64EC }; @@ -2075,6 +1970,113 @@ namespace { } } + struct _Count_traits_8 : _Find_traits_8 { +#ifndef _M_ARM64EC + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi64(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi64(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m128i _Rx6 = _mm256_extracti128_si256(_Val, 0); + const __m128i _Rx7 = _mm256_extracti128_si256(_Val, 1); +#ifdef _M_IX86 + return _mm_cvtsi128_si32(_Rx6) + _mm_extract_epi32(_Rx6, 2) // + + _mm_cvtsi128_si32(_Rx7) + _mm_extract_epi32(_Rx7, 2); +#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv + return _mm_cvtsi128_si64(_Rx6) + _mm_extract_epi64(_Rx6, 1) // + + _mm_cvtsi128_si64(_Rx7) + _mm_extract_epi64(_Rx7, 1); +#endif // ^^^ defined(_M_X64) ^^^ + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { +#ifdef _M_IX86 + return _mm_cvtsi128_si32(_Val) + _mm_extract_epi32(_Val, 2); +#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv + return _mm_cvtsi128_si64(_Val) + _mm_extract_epi64(_Val, 1); +#endif // ^^^ defined(_M_X64) ^^^ + } +#endif // !_M_ARM64EC + }; + + struct _Count_traits_4 : _Find_traits_4 { +#ifndef _M_ARM64EC + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi32(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi32(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane + const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+3),0,0,0 + return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) + + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Rx4 = _mm_hadd_epi32(_Val, _mm_setzero_si128()); // (0+1),(2+3),0,0 + const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+3),0,0,0 + return _mm_cvtsi128_si32(_Rx5); + } +#endif // !_M_ARM64EC + }; + + struct _Count_traits_2 : _Find_traits_2 { +#ifndef _M_ARM64EC + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi16(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi16(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Rx2 = _mm256_hadd_epi16(_Val, _mm256_setzero_si256()); // (0+1),..,(6+7),0,0,0,0 per lane + const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend + return _Count_traits_4::_Reduce_avx(_Rx3); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Rx2 = _mm_hadd_epi16(_Val, _mm_setzero_si128()); // (0+1),..,(6+7),0,0,0,0 + const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend + return _Count_traits_4::_Reduce_sse(_Rx3); + } +#endif // !_M_ARM64EC + }; + + struct _Count_traits_1 : _Find_traits_1 { +#ifndef _M_ARM64EC + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_sub_epi8(_Lhs, _Rhs); + } + + static __m128i _Sub_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { + return _mm_sub_epi8(_Lhs, _Rhs); + } + + static size_t _Reduce_avx(const __m256i _Val) noexcept { + const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); + const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); + const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane + return _Count_traits_2::_Reduce_avx(_Rx1); + } + + static size_t _Reduce_sse(const __m128i _Val) noexcept { + const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); + const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); + const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) + return _Count_traits_2::_Reduce_sse(_Rx1); + } +#endif // !_M_ARM64EC + }; + template __declspec(noalias) size_t __stdcall __std_count_trivial_impl(const void* _First, const void* const _Last, const _Ty _Val) noexcept { @@ -2688,22 +2690,22 @@ const void* __stdcall __std_find_last_trivial_8( __declspec(noalias) size_t __stdcall __std_count_trivial_1(const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { - return __std_count_trivial_impl<_Find_traits_1>(_First, _Last, _Val); + return __std_count_trivial_impl<_Count_traits_1>(_First, _Last, _Val); } __declspec(noalias) size_t __stdcall __std_count_trivial_2(const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { - return __std_count_trivial_impl<_Find_traits_2>(_First, _Last, _Val); + return __std_count_trivial_impl<_Count_traits_2>(_First, _Last, _Val); } __declspec(noalias) size_t __stdcall __std_count_trivial_4(const void* const _First, const void* const _Last, const uint32_t _Val) noexcept { - return __std_count_trivial_impl<_Find_traits_4>(_First, _Last, _Val); + return __std_count_trivial_impl<_Count_traits_4>(_First, _Last, _Val); } __declspec(noalias) size_t __stdcall __std_count_trivial_8(const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { - return __std_count_trivial_impl<_Find_traits_8>(_First, _Last, _Val); + return __std_count_trivial_impl<_Count_traits_8>(_First, _Last, _Val); } const void* __stdcall __std_find_first_of_trivial_1( From 403ef98cf1709bdf4a641606583be121f4136862 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 10:32:49 +0300 Subject: [PATCH 09/27] formatting --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 252aaeb9c27..2369dd59721 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1970,9 +1970,9 @@ namespace { } } - struct _Count_traits_8 : _Find_traits_8 { + struct _Count_traits_8 : _Find_traits_8 { #ifndef _M_ARM64EC - static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi64(_Lhs, _Rhs); } From 983e93a6668490c0129986e90b09d21f520c1c72 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 11:10:02 +0300 Subject: [PATCH 10/27] Stand away from overflows! --- stl/src/vector_algorithms.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2369dd59721..2726f3df85f 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2004,6 +2004,8 @@ namespace { struct _Count_traits_4 : _Find_traits_4 { #ifndef _M_ARM64EC + static constexpr size_t _Max_count = 0x1FFF'FFFF; + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi32(_Lhs, _Rhs); } @@ -2029,6 +2031,8 @@ namespace { struct _Count_traits_2 : _Find_traits_2 { #ifndef _M_ARM64EC + static constexpr size_t _Max_count = 0x7FFF; + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi16(_Lhs, _Rhs); } @@ -2053,6 +2057,8 @@ namespace { struct _Count_traits_1 : _Find_traits_1 { #ifndef _M_ARM64EC + static constexpr size_t _Max_count = 0xFF; + static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi8(_Lhs, _Rhs); } @@ -2064,14 +2070,14 @@ namespace { static size_t _Reduce_avx(const __m256i _Val) noexcept { const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane + const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane return _Count_traits_2::_Reduce_avx(_Rx1); } static size_t _Reduce_sse(const __m128i _Val) noexcept { const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); - const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) + const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) return _Count_traits_2::_Reduce_sse(_Rx1); } #endif // !_M_ARM64EC @@ -2094,7 +2100,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Avx_size); } else { - constexpr size_t _Max_portion_size = ((size_t{1} << (sizeof(_Ty) * 8)) - 2) * 32 / sizeof(_Ty); + constexpr size_t _Max_portion_size = (_Traits::_Max_count - 1) * (32 / sizeof(_Ty)); const size_t _Portion_size = _Avx_size < _Max_portion_size ? _Avx_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Avx_size -= _Portion_size; @@ -2143,7 +2149,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Sse_size); } else { - constexpr size_t _Max_portion_size = ((size_t{1} << (sizeof(_Ty) * 8)) - 1) * 16 / sizeof(_Ty); + constexpr size_t _Max_portion_size = _Traits::_Max_count * (16 / sizeof(_Ty)); const size_t _Portion_size = _Sse_size < _Max_portion_size ? _Sse_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Sse_size -= _Portion_size; From 64b6fe624e24c7a4687dd821b44e3f163789d67f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 11:22:12 +0300 Subject: [PATCH 11/27] sizes are bytes --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2726f3df85f..406a3e906d2 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2100,7 +2100,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Avx_size); } else { - constexpr size_t _Max_portion_size = (_Traits::_Max_count - 1) * (32 / sizeof(_Ty)); + constexpr size_t _Max_portion_size = (_Traits::_Max_count - 1) * 32; const size_t _Portion_size = _Avx_size < _Max_portion_size ? _Avx_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Avx_size -= _Portion_size; @@ -2149,7 +2149,7 @@ namespace { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Sse_size); } else { - constexpr size_t _Max_portion_size = _Traits::_Max_count * (16 / sizeof(_Ty)); + constexpr size_t _Max_portion_size = _Traits::_Max_count * 16; const size_t _Portion_size = _Sse_size < _Max_portion_size ? _Sse_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Sse_size -= _Portion_size; From 85263d864823eea7241ae85691b261b66027abab Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 11:30:09 +0300 Subject: [PATCH 12/27] counting overflow better coverage --- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index d762e6b4b43..f9ccab434c5 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -104,13 +104,8 @@ void test_count(mt19937_64& gen) { template void test_count_zero() { // text that counters don't overflow - vector input; - input.reserve(dataCount); + vector input(1000000, T{0}); test_case_count(input, T{0}); - for (size_t attempts = 0; attempts < dataCount; ++attempts) { - input.push_back(0); - test_case_count(input, T{0}); - } } template From 0a92efcf36ca00a6e8f28aa33f8e7e20aca415ee Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 12:52:33 +0300 Subject: [PATCH 13/27] fewer ops to reduce --- stl/src/vector_algorithms.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 406a3e906d2..2874d339d30 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1981,14 +1981,13 @@ namespace { } static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m128i _Rx6 = _mm256_extracti128_si256(_Val, 0); - const __m128i _Rx7 = _mm256_extracti128_si256(_Val, 1); + const __m128i _Lo64 = _mm256_extracti128_si256(_Val, 0); + const __m128i _Hi64 = _mm256_extracti128_si256(_Val, 1); + const __m128i _Rx8 = _mm_add_epi64(_Lo64, _Hi64); #ifdef _M_IX86 - return _mm_cvtsi128_si32(_Rx6) + _mm_extract_epi32(_Rx6, 2) // - + _mm_cvtsi128_si32(_Rx7) + _mm_extract_epi32(_Rx7, 2); + return _mm_cvtsi128_si32(_Rx8) + _mm_extract_epi32(_Rx8, 2); #else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv - return _mm_cvtsi128_si64(_Rx6) + _mm_extract_epi64(_Rx6, 1) // - + _mm_cvtsi128_si64(_Rx7) + _mm_extract_epi64(_Rx7, 1); + return _mm_cvtsi128_si64(_Rx8) + _mm_extract_epi64(_Rx8, 1); #endif // ^^^ defined(_M_X64) ^^^ } @@ -2015,10 +2014,12 @@ namespace { } static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane - const __m256i _Rx5 = _mm256_hadd_epi32(_Rx4, _mm256_setzero_si256()); // (0+3),0,0,0 - return _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 0)) - + _mm_cvtsi128_si32(_mm256_extracti128_si256(_Rx5, 1)); + constexpr auto _Shuf = _MM_SHUFFLE(3, 1, 2, 0); // Cross lane, to reduce further on low lane + const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane + const __m256i _Rx5 = _mm256_permute4x64_epi64(_Rx4, _Shuf); // low lane (0+1),(2+3),(4+5),(6+7) + const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+3),(4+7),0,0 + const __m256i _Rx7 = _mm256_hadd_epi32(_Rx6, _mm256_setzero_si256()); // (0+7),0,0,0 + return _mm_cvtsi128_si32(_mm256_castsi256_si128(_Rx7)); } static size_t _Reduce_sse(const __m128i _Val) noexcept { From d816df0e062a43e025eecd5e6191b9775d651b7f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 18:29:03 +0300 Subject: [PATCH 14/27] Comments cleanup --- stl/src/vector_algorithms.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2874d339d30..541914f3a9b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2017,14 +2017,14 @@ namespace { constexpr auto _Shuf = _MM_SHUFFLE(3, 1, 2, 0); // Cross lane, to reduce further on low lane const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane const __m256i _Rx5 = _mm256_permute4x64_epi64(_Rx4, _Shuf); // low lane (0+1),(2+3),(4+5),(6+7) - const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+3),(4+7),0,0 - const __m256i _Rx7 = _mm256_hadd_epi32(_Rx6, _mm256_setzero_si256()); // (0+7),0,0,0 + const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+..+3),(4+...+7),0,0 + const __m256i _Rx7 = _mm256_hadd_epi32(_Rx6, _mm256_setzero_si256()); // (0+...+7),0,0,0 return _mm_cvtsi128_si32(_mm256_castsi256_si128(_Rx7)); } static size_t _Reduce_sse(const __m128i _Val) noexcept { const __m128i _Rx4 = _mm_hadd_epi32(_Val, _mm_setzero_si128()); // (0+1),(2+3),0,0 - const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+3),0,0,0 + const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+3),0,0,0 return _mm_cvtsi128_si32(_Rx5); } #endif // !_M_ARM64EC @@ -2043,14 +2043,14 @@ namespace { } static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Rx2 = _mm256_hadd_epi16(_Val, _mm256_setzero_si256()); // (0+1),..,(6+7),0,0,0,0 per lane - const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); // zero extend + const __m256i _Rx2 = _mm256_hadd_epi16(_Val, _mm256_setzero_si256()); + const __m256i _Rx3 = _mm256_unpacklo_epi16(_Rx2, _mm256_setzero_si256()); return _Count_traits_4::_Reduce_avx(_Rx3); } static size_t _Reduce_sse(const __m128i _Val) noexcept { - const __m128i _Rx2 = _mm_hadd_epi16(_Val, _mm_setzero_si128()); // (0+1),..,(6+7),0,0,0,0 - const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); // zero extend + const __m128i _Rx2 = _mm_hadd_epi16(_Val, _mm_setzero_si128()); + const __m128i _Rx3 = _mm_unpacklo_epi16(_Rx2, _mm_setzero_si128()); return _Count_traits_4::_Reduce_sse(_Rx3); } #endif // !_M_ARM64EC @@ -2071,14 +2071,14 @@ namespace { static size_t _Reduce_avx(const __m256i _Val) noexcept { const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) per lane + const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); return _Count_traits_2::_Reduce_avx(_Rx1); } static size_t _Reduce_sse(const __m128i _Val) noexcept { const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); - const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); // (0+1),..,(15+16) + const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); return _Count_traits_2::_Reduce_sse(_Rx1); } #endif // !_M_ARM64EC From dedc0cc254f211370c7cb76d73ec15c06a96fd33 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 22 Apr 2024 18:35:42 +0300 Subject: [PATCH 15/27] reduce 1-byte with `sad` instruction --- stl/src/vector_algorithms.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 541914f3a9b..1b8e470a673 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2069,17 +2069,13 @@ namespace { } static size_t _Reduce_avx(const __m256i _Val) noexcept { - const __m256i _Hi8 = _mm256_unpackhi_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Lo8 = _mm256_unpacklo_epi8(_Val, _mm256_setzero_si256()); - const __m256i _Rx1 = _mm256_hadd_epi16(_Lo8, _Hi8); - return _Count_traits_2::_Reduce_avx(_Rx1); + const __m256i _Rx1 = _mm256_sad_epu8(_Val, _mm256_setzero_si256()); + return _Count_traits_8::_Reduce_avx(_Rx1); } static size_t _Reduce_sse(const __m128i _Val) noexcept { - const __m128i _Hi8 = _mm_unpackhi_epi8(_Val, _mm_setzero_si128()); - const __m128i _Lo8 = _mm_unpacklo_epi8(_Val, _mm_setzero_si128()); - const __m128i _Rx1 = _mm_hadd_epi16(_Lo8, _Hi8); - return _Count_traits_2::_Reduce_sse(_Rx1); + const __m128i _Rx1 = _mm_sad_epu8(_Val, _mm_setzero_si128()); + return _Count_traits_8::_Reduce_sse(_Rx1); } #endif // !_M_ARM64EC }; From 015d4f7a4a6ee161ca55053be4716e6ebc7eabba Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 24 Apr 2024 15:52:46 -0700 Subject: [PATCH 16/27] Simplify `_Count_traits_8::_Reduce_avx()` by reusing `_Reduce_sse()`. --- stl/src/vector_algorithms.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 1b8e470a673..0ff23fd1153 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1984,11 +1984,7 @@ namespace { const __m128i _Lo64 = _mm256_extracti128_si256(_Val, 0); const __m128i _Hi64 = _mm256_extracti128_si256(_Val, 1); const __m128i _Rx8 = _mm_add_epi64(_Lo64, _Hi64); -#ifdef _M_IX86 - return _mm_cvtsi128_si32(_Rx8) + _mm_extract_epi32(_Rx8, 2); -#else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv - return _mm_cvtsi128_si64(_Rx8) + _mm_extract_epi64(_Rx8, 1); -#endif // ^^^ defined(_M_X64) ^^^ + return _Reduce_sse(_Rx8); } static size_t _Reduce_sse(const __m128i _Val) noexcept { From 7e39a04c340e65da4043b1837c9c1b90f95adbf0 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 24 Apr 2024 17:17:24 -0700 Subject: [PATCH 17/27] Fix `_Count_traits_4::_Max_count`. 0x1FFF'FFFF would be strict for the SSE4.2 codepath, but 0xFFF'FFFF is strict for the AVX2 codepath. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0ff23fd1153..47455e20be7 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1999,7 +1999,7 @@ namespace { struct _Count_traits_4 : _Find_traits_4 { #ifndef _M_ARM64EC - static constexpr size_t _Max_count = 0x1FFF'FFFF; + static constexpr size_t _Max_count = 0xFFF'FFFF; static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi32(_Lhs, _Rhs); From 8d4aab51ee54932ccd651266ae70b33b7f547bcd Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 02:52:44 -0700 Subject: [PATCH 18/27] Add detailed comments explaining each `_Max_count`. --- stl/src/vector_algorithms.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 47455e20be7..3e8deca020e 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1999,6 +1999,16 @@ namespace { struct _Count_traits_4 : _Find_traits_4 { #ifndef _M_ARM64EC + // For both AVX2 and SSE4.2, the reduced result is extracted by _mm_cvtsi128_si32 + // as a signed 32-bit integer, so we can count at most 2^31 - 1 elements (0x7FFF'FFFF). + + // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, i.e. _Max_count * 8 elements. + // Therefore, _Max_count is 0x7FFF'FFFF / 8 which truncates to 0xFFF'FFFF. + + // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, i.e. _Max_count * 4 elements. + // Therefore, _Max_count could be 0x7FFF'FFFF / 4 which truncates to 0x1FFF'FFFF, + // but it's simpler to use the smaller bound for both codepaths. + static constexpr size_t _Max_count = 0xFFF'FFFF; static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { @@ -2028,6 +2038,14 @@ namespace { struct _Count_traits_2 : _Find_traits_2 { #ifndef _M_ARM64EC + // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, i.e. _Max_count * 16 elements. + // _mm256_hadd_epi16 processes signed 16-bit integers, and 16 of those fit in 256 bits. + + // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, i.e. _Max_count * 8 elements. + // _mm_hadd_epi16 processes signed 16-bit integers, and 8 of those fit in 128 bits. + + // For both codepaths, this is why _Max_count is the maximum signed 16-bit integer. + static constexpr size_t _Max_count = 0x7FFF; static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { @@ -2054,6 +2072,14 @@ namespace { struct _Count_traits_1 : _Find_traits_1 { #ifndef _M_ARM64EC + // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, and we have 1-byte elements. + // _mm256_sad_epu8 processes packed unsigned 8-bit integers, and 32 of those fit in 256 bits. + + // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, and we have 1-byte elements. + // _mm_sad_epu8 processes packed unsigned 8-bit integers, and 16 of those fit in 128 bits. + + // For both codepaths, this is why _Max_count is the maximum unsigned 8-bit integer. + static constexpr size_t _Max_count = 0xFF; static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { From 50d610a35aa50379f4b51ca7bad45371a2b18b33 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 03:16:00 -0700 Subject: [PATCH 19/27] For clarity, scope `__m128i _Count_vector` to each iteration of the SSE4.2 loop. It never carries information across iterations. --- stl/src/vector_algorithms.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 3e8deca020e..8a7c9e97ef9 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2162,7 +2162,6 @@ namespace { } else if (size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { const __m128i _Comparand = _Traits::_Set_sse(_Val); const void* _Stop_at = _First; - __m128i _Count_vector = _mm_setzero_si128(); for (;;) { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { @@ -2174,6 +2173,8 @@ namespace { _Sse_size -= _Portion_size; } + __m128i _Count_vector = _mm_setzero_si128(); + do { const __m128i _Data = _mm_loadu_si128(static_cast(_First)); const __m128i _Mask = _Traits::_Cmp_sse(_Data, _Comparand); @@ -2189,8 +2190,6 @@ namespace { if (_Sse_size == 0) { break; } - - _Count_vector = _mm_setzero_si128(); } } } From c06d4ff41356f96b09a253db3e9884d850bf2133 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 04:13:22 -0700 Subject: [PATCH 20/27] For the AVX2 loop, scope `__m256i _Count_vector` separately for the main loop and tail. This allows us to avoid subtracting 1 from `_Max_count`, making it more similar to the SSE4.2 loop. --- stl/src/vector_algorithms.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 8a7c9e97ef9..9959405eace 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2113,18 +2113,19 @@ namespace { if (size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _First; - __m256i _Count_vector = _mm256_setzero_si256(); for (;;) { if constexpr (sizeof(_Ty) >= sizeof(size_t)) { _Advance_bytes(_Stop_at, _Avx_size); } else { - constexpr size_t _Max_portion_size = (_Traits::_Max_count - 1) * 32; + constexpr size_t _Max_portion_size = _Traits::_Max_count * 32; const size_t _Portion_size = _Avx_size < _Max_portion_size ? _Avx_size : _Max_portion_size; _Advance_bytes(_Stop_at, _Portion_size); _Avx_size -= _Portion_size; } + __m256i _Count_vector = _mm256_setzero_si256(); + do { const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); @@ -2132,28 +2133,27 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); + _Result += _Traits::_Reduce_avx(_Count_vector); + if constexpr (sizeof(_Ty) >= sizeof(size_t)) { break; } else { if (_Avx_size == 0) { break; } - - _Result += _Traits::_Reduce_avx(_Count_vector); - _Count_vector = _mm256_setzero_si256(); } } if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + __m256i _Count_vector = _mm256_setzero_si256(); const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); _Count_vector = _Traits::_Sub_avx(_Count_vector, _mm256_and_si256(_Mask, _Tail_mask)); _Advance_bytes(_First, _Avx_tail_size); + _Result += _Traits::_Reduce_avx(_Count_vector); } - _Result += _Traits::_Reduce_avx(_Count_vector); - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 if constexpr (sizeof(_Ty) >= 4) { From 3f958151faa16b9e156394119220ed1f78f3f188 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 04:24:58 -0700 Subject: [PATCH 21/27] Fix comment typo. --- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index f9ccab434c5..2d3e86103e3 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -103,7 +103,7 @@ void test_count(mt19937_64& gen) { } template -void test_count_zero() { // text that counters don't overflow +void test_count_zero() { // test that counters don't overflow vector input(1000000, T{0}); test_case_count(input, T{0}); } From 22b561b0bd51dcd68784ea2a335f80eecf1338ca Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 04:59:36 -0700 Subject: [PATCH 22/27] Change test_count_zero() to test more lengths. --- .../VSO_0000000_vector_algorithms/test.cpp | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 2d3e86103e3..07b90817063 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -89,6 +89,17 @@ void test_case_count(const vector& input, T v) { #endif // _HAS_CXX20 } +template +void test_count_zero(const vector& input, const ptrdiff_t n) { + const auto first = input.begin(); + const auto last = first + n; + + assert(count(first, last, T{0}) == n); +#if _HAS_CXX20 + assert(ranges::count(first, last, T{0}) == n); +#endif // _HAS_CXX20 +} + template void test_count(mt19937_64& gen) { using TD = conditional_t; @@ -100,12 +111,27 @@ void test_count(mt19937_64& gen) { input.push_back(static_cast(dis(gen))); test_case_count(input, static_cast(dis(gen))); } -} -template -void test_count_zero() { // test that counters don't overflow - vector input(1000000, T{0}); - test_case_count(input, T{0}); + { + input.assign(1'000'000, T{0}); + + // test that counters don't overflow + test_count_zero(input, 1'000'000); + + // Test the AVX2 maximum portion followed by all possible tail lengths, for 1-byte and 2-byte elements. + // It's okay to test these lengths for other elements, or other instruction sets. + for (ptrdiff_t i = 8'160; i < 8'192; ++i) { + test_count_zero(input, i); + } + + for (ptrdiff_t i = 524'272; i < 524'288; ++i) { + test_count_zero(input, i); + } + + // Test a random length. + uniform_int_distribution len(0, 999'999); + test_count_zero(input, len(gen)); + } } template @@ -748,16 +774,6 @@ void test_vector_algorithms(mt19937_64& gen) { test_count(gen); test_count(gen); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_count_zero(); - test_find(gen); test_find(gen); test_find(gen); From e587698929a2c54475742fbff9f74bb9f7e8d7dc Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 25 Apr 2024 16:43:08 +0300 Subject: [PATCH 23/27] Restore popcnt approach --- stl/src/vector_algorithms.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 9959405eace..62f9c9cac07 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2145,13 +2145,12 @@ namespace { } if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { - __m256i _Count_vector = _mm256_setzero_si256(); const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); - const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); - _Count_vector = _Traits::_Sub_avx(_Count_vector, _mm256_and_si256(_Mask, _Tail_mask)); + const __m256i _Mask = _mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask); + const size_t _Tail_count = __popcnt(_mm256_movemask_epi8(_Mask)) / sizeof(_Ty); _Advance_bytes(_First, _Avx_tail_size); - _Result += _Traits::_Reduce_avx(_Count_vector); + _Result += _Tail_count; } _mm256_zeroupper(); // TRANSITION, DevCom-10331414 From 87749d22ba40c67e38c4b8a84a8ca811e5e5b0e4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 25 Apr 2024 17:14:17 +0300 Subject: [PATCH 24/27] Get my bounds back --- stl/src/vector_algorithms.cpp | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 62f9c9cac07..39e47ae5d3d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1989,7 +1989,7 @@ namespace { static size_t _Reduce_sse(const __m128i _Val) noexcept { #ifdef _M_IX86 - return _mm_cvtsi128_si32(_Val) + _mm_extract_epi32(_Val, 2); + return static_cast(_mm_cvtsi128_si32(_Val)) + static_cast(_mm_extract_epi32(_Val, 2)); #else // ^^^ defined(_M_IX86) / defined(_M_X64) vvv return _mm_cvtsi128_si64(_Val) + _mm_extract_epi64(_Val, 1); #endif // ^^^ defined(_M_X64) ^^^ @@ -1999,17 +1999,14 @@ namespace { struct _Count_traits_4 : _Find_traits_4 { #ifndef _M_ARM64EC - // For both AVX2 and SSE4.2, the reduced result is extracted by _mm_cvtsi128_si32 - // as a signed 32-bit integer, so we can count at most 2^31 - 1 elements (0x7FFF'FFFF). + // For AVX2, we reduce using hadd_epi32 3 times, therefore to avoid oveflow _Max_count is 0x1FFF'FFFF, + // which is multiplied by two 3 times 0xFFFF'FFF8, a greater limit would overflow - // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, i.e. _Max_count * 8 elements. - // Therefore, _Max_count is 0x7FFF'FFFF / 8 which truncates to 0xFFF'FFFF. - - // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, i.e. _Max_count * 4 elements. - // Therefore, _Max_count could be 0x7FFF'FFFF / 4 which truncates to 0x1FFF'FFFF, + // For AVX2, we reduce using hadd_epi32 2 times, therefore to avoid oveflow _Max_count is 0x3FFF'FFFF, + // which is multiplied by two 2 times 0xFFFF'FFFC, a greater limit would overflow // but it's simpler to use the smaller bound for both codepaths. - static constexpr size_t _Max_count = 0xFFF'FFFF; + static constexpr size_t _Max_count = 0x1FFF'FFFF; static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { return _mm256_sub_epi32(_Lhs, _Rhs); @@ -2025,26 +2022,22 @@ namespace { const __m256i _Rx5 = _mm256_permute4x64_epi64(_Rx4, _Shuf); // low lane (0+1),(2+3),(4+5),(6+7) const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+..+3),(4+...+7),0,0 const __m256i _Rx7 = _mm256_hadd_epi32(_Rx6, _mm256_setzero_si256()); // (0+...+7),0,0,0 - return _mm_cvtsi128_si32(_mm256_castsi256_si128(_Rx7)); + return static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(_Rx7))); } static size_t _Reduce_sse(const __m128i _Val) noexcept { const __m128i _Rx4 = _mm_hadd_epi32(_Val, _mm_setzero_si128()); // (0+1),(2+3),0,0 const __m128i _Rx5 = _mm_hadd_epi32(_Rx4, _mm_setzero_si128()); // (0+...+3),0,0,0 - return _mm_cvtsi128_si32(_Rx5); + return static_cast(_mm_cvtsi128_si32(_Rx5)); } #endif // !_M_ARM64EC }; struct _Count_traits_2 : _Find_traits_2 { #ifndef _M_ARM64EC - // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, i.e. _Max_count * 16 elements. - // _mm256_hadd_epi16 processes signed 16-bit integers, and 16 of those fit in 256 bits. - - // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, i.e. _Max_count * 8 elements. - // _mm_hadd_epi16 processes signed 16-bit integers, and 8 of those fit in 128 bits. - - // For both codepaths, this is why _Max_count is the maximum signed 16-bit integer. + // For both AVX2 and SSE2 we need to fit elements after hadd_epi16 to 16 bits, + // therefore _Max_count is 0x7FFF, which is when added to itself is 0xFFFE, + // a greater limit would overflow. static constexpr size_t _Max_count = 0x7FFF; From 2e1d6c733a9d6a15607177dd7ee3343c63bd7019 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 25 Apr 2024 17:20:07 +0300 Subject: [PATCH 25/27] typos --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 39e47ae5d3d..e0aba2726c3 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2002,7 +2002,7 @@ namespace { // For AVX2, we reduce using hadd_epi32 3 times, therefore to avoid oveflow _Max_count is 0x1FFF'FFFF, // which is multiplied by two 3 times 0xFFFF'FFF8, a greater limit would overflow - // For AVX2, we reduce using hadd_epi32 2 times, therefore to avoid oveflow _Max_count is 0x3FFF'FFFF, + // For SSE2, we reduce using hadd_epi32 2 times, therefore to avoid oveflow _Max_count is 0x3FFF'FFFF, // which is multiplied by two 2 times 0xFFFF'FFFC, a greater limit would overflow // but it's simpler to use the smaller bound for both codepaths. @@ -2020,7 +2020,7 @@ namespace { constexpr auto _Shuf = _MM_SHUFFLE(3, 1, 2, 0); // Cross lane, to reduce further on low lane const __m256i _Rx4 = _mm256_hadd_epi32(_Val, _mm256_setzero_si256()); // (0+1),(2+3),0,0 per lane const __m256i _Rx5 = _mm256_permute4x64_epi64(_Rx4, _Shuf); // low lane (0+1),(2+3),(4+5),(6+7) - const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+..+3),(4+...+7),0,0 + const __m256i _Rx6 = _mm256_hadd_epi32(_Rx5, _mm256_setzero_si256()); // (0+...+3),(4+...+7),0,0 const __m256i _Rx7 = _mm256_hadd_epi32(_Rx6, _mm256_setzero_si256()); // (0+...+7),0,0,0 return static_cast(_mm_cvtsi128_si32(_mm256_castsi256_si128(_Rx7))); } From fc6dcbd862d8f516dc25a2ba46020ae2920e495b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 25 Apr 2024 18:53:26 +0300 Subject: [PATCH 26/27] restore SSE4.2 comment --- stl/src/vector_algorithms.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index e0aba2726c3..a6c69e0fb2a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2141,9 +2141,10 @@ namespace { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); const __m256i _Mask = _mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask); - const size_t _Tail_count = __popcnt(_mm256_movemask_epi8(_Mask)) / sizeof(_Ty); + const int _Bingo = _mm256_movemask_epi8(_Mask); + const size_t _Tail_count = __popcnt(_Bingo); // Assume available with SSE4.2 + _Result += _Tail_count / sizeof(_Ty); _Advance_bytes(_First, _Avx_tail_size); - _Result += _Tail_count; } _mm256_zeroupper(); // TRANSITION, DevCom-10331414 From c5dd5c22f02a4ca6134bbd2e8e123bab2fcc10c7 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 25 Apr 2024 10:53:46 -0700 Subject: [PATCH 27/27] Clarify comments. --- stl/src/vector_algorithms.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index a6c69e0fb2a..d7fa370ec6d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1999,12 +1999,11 @@ namespace { struct _Count_traits_4 : _Find_traits_4 { #ifndef _M_ARM64EC - // For AVX2, we reduce using hadd_epi32 3 times, therefore to avoid oveflow _Max_count is 0x1FFF'FFFF, - // which is multiplied by two 3 times 0xFFFF'FFF8, a greater limit would overflow + // For AVX2, we use hadd_epi32 three times to combine pairs of 32-bit counters into 32-bit results. + // Therefore, _Max_count is 0x1FFF'FFFF, which is 0xFFFF'FFF8 when doubled three times; any more would overflow. - // For SSE2, we reduce using hadd_epi32 2 times, therefore to avoid oveflow _Max_count is 0x3FFF'FFFF, - // which is multiplied by two 2 times 0xFFFF'FFFC, a greater limit would overflow - // but it's simpler to use the smaller bound for both codepaths. + // For SSE4.2, we use hadd_epi32 twice. This would allow a larger limit, + // but it's simpler to use the smaller limit for both codepaths. static constexpr size_t _Max_count = 0x1FFF'FFFF; @@ -2035,9 +2034,8 @@ namespace { struct _Count_traits_2 : _Find_traits_2 { #ifndef _M_ARM64EC - // For both AVX2 and SSE2 we need to fit elements after hadd_epi16 to 16 bits, - // therefore _Max_count is 0x7FFF, which is when added to itself is 0xFFFE, - // a greater limit would overflow. + // For both AVX2 and SSE4.2, we use hadd_epi16 once to combine pairs of 16-bit counters into 16-bit results. + // Therefore, _Max_count is 0x7FFF, which is 0xFFFE when doubled; any more would overflow. static constexpr size_t _Max_count = 0x7FFF; @@ -2066,12 +2064,13 @@ namespace { struct _Count_traits_1 : _Find_traits_1 { #ifndef _M_ARM64EC // For AVX2, _Max_portion_size below is _Max_count * 32 bytes, and we have 1-byte elements. - // _mm256_sad_epu8 processes packed unsigned 8-bit integers, and 32 of those fit in 256 bits. + // We're using packed 8-bit counters, and 32 of those fit in 256 bits. // For SSE4.2, _Max_portion_size below is _Max_count * 16 bytes, and we have 1-byte elements. - // _mm_sad_epu8 processes packed unsigned 8-bit integers, and 16 of those fit in 128 bits. + // We're using packed 8-bit counters, and 16 of those fit in 128 bits. // For both codepaths, this is why _Max_count is the maximum unsigned 8-bit integer. + // (The reduction steps aren't the limiting factor here.) static constexpr size_t _Max_count = 0xFF;