From b4aceb2ec4b404c183e879a23c51b9a30ca233b3 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 14:45:40 +0300 Subject: [PATCH 01/18] replace replace --- benchmarks/src/replace.cpp | 15 ++++ stl/inc/algorithm | 67 ++++++++++++++++++ stl/inc/xutility | 40 ++++++----- stl/src/vector_algorithms.cpp | 68 +++++++++++++++++++ .../VSO_0000000_vector_algorithms/test.cpp | 49 +++++++++++++ 5 files changed, 221 insertions(+), 18 deletions(-) diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index 8fc66ab7841..abe64a748be 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -37,6 +37,18 @@ const char src[] = "euismod eros, ut posuere ligula ullamcorper id. Nullam aliquam malesuada est at dignissim. Pellentesque finibus " "sagittis libero nec bibendum. Phasellus dolor ipsum, finibus quis turpis quis, mollis interdum felis."; +template +void r(benchmark::State& state) { + const std::vector a(std::begin(src), std::end(src)); + std::vector b(std::size(src)); + + for (auto _ : state) { + b = a; + std::replace(std::begin(b), std::end(b), T{'m'}, T{'w'}); + } +} + + template void rc(benchmark::State& state) { const std::vector a(std::begin(src), std::end(src)); @@ -58,6 +70,9 @@ void rc_if(benchmark::State& state) { } } +BENCHMARK(r); +BENCHMARK(r); + BENCHMARK(rc); BENCHMARK(rc); BENCHMARK(rc); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index ff1f8b65e3c..8e8f2a25082 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -73,6 +73,12 @@ __declspec(noalias) _Min_max_8i __stdcall __std_minmax_8i(const void* _First, co __declspec(noalias) _Min_max_8u __stdcall __std_minmax_8u(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_f __stdcall __std_minmax_f(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* _First, const void* _Last) noexcept; + +// TRANSITION, DevCom-10610477 +__declspec(noalias) void __stdcall __std_replace_trivial_4( + void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_trivial_8( + void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; } // extern "C" _STD_BEGIN @@ -180,6 +186,26 @@ _Ty1* __std_find_first_of_trivial( } } +template +__declspec(noalias) void _Replace_trivial( + _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { + if constexpr (is_pointer_v<_Ty>) { +#ifdef _WIN64 + ::__std_replace_trivial_8( + _First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); +#else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv + ::__std_replace_trivial_4( + _First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); +#endif // ^^^ !defined(_WIN64) ^^^ + } else if constexpr (sizeof(_Ty) == 4) { + ::__std_replace_trivial_4(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 8) { + ::__std_replace_trivial_8(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); + } else { + static_assert(_Always_false<_Ty>, "Unexpected size"); + } +} + // find_first_of vectorization is likely to be a win after this size (in elements) _INLINE_VAR constexpr ptrdiff_t _Threshold_find_first_of = 16; @@ -188,6 +214,13 @@ template _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr> // can replace value comparison with bitwise comparison && sizeof(_Iter_value_t<_It1>) <= 2; // pcmpestri compatible size + +// Can we activate the vector algorithms for replace? +template +_INLINE_VAR constexpr bool _Vector_alg_in_replace_is_safe = + _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value + && sizeof(_Iter_value_t<_Iter>) >= 4 // avx masked op compatible size + && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -3807,6 +3840,22 @@ _CONSTEXPR20 void replace(const _FwdIt _First, const _FwdIt _Last, const _Ty& _O _STD _Adl_verify_range(_First, _Last); auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); + +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Vector_alg_in_replace_is_safe<_FwdIt, _Ty, _Ty>) { +#if _HAS_CXX20 + if (!_STD is_constant_evaluated()) +#endif // _HAS_CXX20 + { + if (_STD _Could_compare_equal_to_value_type<_FwdIt>(_Oldval)) { + _STD _Replace_trivial(_STD _To_address(_UFirst), _STD _To_address(_ULast), _Oldval, _Newval); + } + + return; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + for (; _UFirst != _ULast; ++_UFirst) { if (*_UFirst == _Oldval) { *_UFirst = _Newval; @@ -3860,6 +3909,24 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(indirectly_writable<_It, const _Ty2&>); _STL_INTERNAL_STATIC_ASSERT(indirect_binary_predicate, const _Ty1*>); +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (is_same_v<_Pj, identity> && _Vector_alg_in_replace_is_safe<_It, _Ty1, _Ty2> + && sized_sentinel_for<_Se, _It>) { + if (!_STD is_constant_evaluated()) + { + const auto _Count = _Last - _First; + + if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { + const auto _First_ptr = _STD to_address(_First); + const auto _Last_ptr = _First_ptr + _Count; + _STD _Replace_trivial(_First_ptr, _Last_ptr, _Oldval, _Newval); + } + + return _First + _Count; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + for (; _First != _Last; ++_First) { if (_STD invoke(_Proj, *_First) == _Oldval) { *_First = _Newval; diff --git a/stl/inc/xutility b/stl/inc/xutility index f2e0482894a..b91965d19bd 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -5830,30 +5830,34 @@ struct _Vector_alg_in_find_is_safe_object_pointers<_Ty1*, _Ty2*> // either _Ty1 is the same as _Ty2 (ignoring cv-qualifiers), or one of the two is void disjunction, remove_cv_t<_Ty2>>, is_void<_Ty1>, is_void<_Ty2>>> {}; +// Can we activate the vector algorithms for a value and container elements +template +_INLINE_VAR constexpr bool _Vector_alg_in_find_is_safe_elem = disjunction_v< +#ifdef __cpp_lib_byte + // We're finding a std::byte in a range of std::byte. + conjunction, is_same<_Elem, byte>>, +#endif // defined(__cpp_lib_byte) + // We're finding an integer in a range of integers. + // This case is the one that requires careful runtime handling in _Could_compare_equal_to_value_type. + conjunction, is_integral<_Elem>>, + // We're finding an (object or function) pointer in a range of pointers of the same type. + conjunction, is_same<_Ty, _Elem>>, + // We're finding a nullptr in a range of (object or function) pointers. + conjunction, is_pointer<_Elem>>, + // We're finding an object pointer in a range of object pointers, and: + // - One of the pointer types is a cv void*. + // - One of the pointer types is a cv1 U* and the other is a cv2 U*. + _Vector_alg_in_find_is_safe_object_pointers<_Ty, _Elem>>; + // Can we activate the vector algorithms for find/count? -template > +template _INLINE_VAR constexpr bool _Vector_alg_in_find_is_safe = // The iterator must be contiguous so we can get raw pointers. _Iterator_is_contiguous<_Iter> // The iterator must not be volatile. && !_Iterator_is_volatile<_Iter> - // And one of the following conditions must be met: - && disjunction_v< -#ifdef __cpp_lib_byte - // We're finding a std::byte in a range of std::byte. - conjunction, is_same<_Elem, byte>>, -#endif // defined(__cpp_lib_byte) - // We're finding an integer in a range of integers. - // This case is the one that requires careful runtime handling in _Could_compare_equal_to_value_type. - conjunction, is_integral<_Elem>>, - // We're finding an (object or function) pointer in a range of pointers of the same type. - conjunction, is_same<_Ty, _Elem>>, - // We're finding a nullptr in a range of (object or function) pointers. - conjunction, is_pointer<_Elem>>, - // We're finding an object pointer in a range of object pointers, and: - // - One of the pointer types is a cv void*. - // - One of the pointer types is a cv1 U* and the other is a cv2 U*. - _Vector_alg_in_find_is_safe_object_pointers<_Ty, _Elem>>; + // The elements and the value of a certain matching types. + && _Vector_alg_in_find_is_safe_elem<_Ty, _Iter_value_t<_Iter>>; template _NODISCARD constexpr bool _Could_compare_equal_to_value_type(const _Ty& _Val) { diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 1c5fc74e809..629f3ee52ae 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2322,6 +2322,74 @@ namespace { extern "C" { +__declspec(noalias) void __stdcall __std_replace_trivial_4( + void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { + if (_Use_avx2()) { + const __m256i _Comparand = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_Old_val)); + const __m256i _Replacement = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_New_val)); + + const size_t _Full_length = _Byte_length(_First, _Last); + + void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Full_length & ~size_t{0x1F}); + + while (_First != _Stop_at) { + const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); + const __m256i _Mask = _mm256_cmpeq_epi32(_Comparand, _Data); + _mm256_maskstore_epi32(reinterpret_cast(_First), _Mask, _Replacement); + + _Advance_bytes(_First, 32); + } + + if (const size_t _Tail_length = _Full_length & 0x1C; _Tail_length != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Tail_length >> 2); + const __m256i _Data = _mm256_maskload_epi32(reinterpret_cast(_First), _Tail_mask); + const __m256i _Mask = _mm256_and_si256(_mm256_cmpeq_epi32(_Comparand, _Data), _Tail_mask); + _mm256_maskstore_epi32(reinterpret_cast(_First), _Mask, _Replacement); + } + } else { + for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { + if (*_Cur == _Old_val) { + *_Cur = _New_val; + } + } + } +} + +__declspec(noalias) void __stdcall __std_replace_trivial_8( + void* _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { + if (_Use_avx2()) { + const __m256i _Comparand = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_Old_val)); + const __m256i _Replacement = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_New_val)); + + const size_t _Full_length = _Byte_length(_First, _Last); + + void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Full_length & ~size_t{0x1F}); + + while (_First != _Stop_at) { + const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); + const __m256i _Mask = _mm256_cmpeq_epi64(_Comparand, _Data); + _mm256_maskstore_epi64(reinterpret_cast(_First), _Mask, _Replacement); + + _Advance_bytes(_First, 32); + } + + if (const size_t _Tail_length = _Full_length & 0x18; _Tail_length != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Tail_length >> 2); + const __m256i _Data = _mm256_maskload_epi64(reinterpret_cast(_First), _Tail_mask); + const __m256i _Mask = _mm256_and_si256(_mm256_cmpeq_epi64(_Comparand, _Data), _Tail_mask); + _mm256_maskstore_epi64(reinterpret_cast(_First), _Mask, _Replacement); + } + } else { + for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { + if (*_Cur == _Old_val) { + *_Cur = _New_val; + } + } + } +} + __declspec(noalias) void __stdcall __std_bitset_to_string_1( char* const _Dest, const void* _Src, size_t _Size_bits, const char _Elem0, const char _Elem1) noexcept { #ifndef _M_ARM64EC diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index e263e59d628..20f8acab89c 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -536,6 +536,45 @@ namespace test_mismatch_sizes_and_alignments { } } // namespace test_mismatch_sizes_and_alignments +template +void last_known_good_replace(FwdIt first, FwdIt last, const T old_val, const T new_val) { + for (; first != last; ++first) { + if (*first == old_val) { + *first = new_val; + } + } +} + +template +void test_case_replace(const vector& input, T old_val, T new_val) { + vector replaced_actual(input); + vector replaced_expected(input); + replace(replaced_actual.begin(), replaced_actual.end(), old_val, new_val); + last_known_good_replace(replaced_expected.begin(), replaced_expected.end(), old_val, new_val); + assert(replaced_expected == replaced_actual); + +#if _HAS_CXX20 + vector replaced_actual_r(input); + ranges::replace(replaced_actual_r, old_val, new_val); + assert(replaced_expected == replaced_actual_r); +#endif // _HAS_CXX20 +} + +template +void test_replace(mt19937_64& gen) { + using TD = conditional_t; + uniform_int_distribution dis(0, 9); + vector input; + + input.reserve(dataCount); + + test_case_replace(input, static_cast(dis(gen)), static_cast(dis(gen))); + for (size_t i = 0; i != dataCount; ++i) { + input.push_back(static_cast(dis(gen))); + test_case_replace(input, static_cast(dis(gen)), static_cast(dis(gen))); + } +} + template void last_known_good_reverse(BidIt first, BidIt last) { for (; first != last && first != --last; ++first) { @@ -728,6 +767,16 @@ void test_vector_algorithms(mt19937_64& gen) { test_mismatch_sizes_and_alignments::test(); test_mismatch_sizes_and_alignments::test(); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_reverse(gen); test_reverse(gen); test_reverse(gen); From b15baf612245761b7998c774a09ec11b345e7494 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 14:56:36 +0300 Subject: [PATCH 02/18] format --- benchmarks/src/replace.cpp | 1 - stl/inc/algorithm | 5 ++--- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index abe64a748be..4c678229575 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -48,7 +48,6 @@ void r(benchmark::State& state) { } } - template void rc(benchmark::State& state) { const std::vector a(std::begin(src), std::end(src)); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 8e8f2a25082..8c0a6c30312 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -3850,7 +3850,7 @@ _CONSTEXPR20 void replace(const _FwdIt _First, const _FwdIt _Last, const _Ty& _O if (_STD _Could_compare_equal_to_value_type<_FwdIt>(_Oldval)) { _STD _Replace_trivial(_STD _To_address(_UFirst), _STD _To_address(_ULast), _Oldval, _Newval); } - + return; } } @@ -3912,8 +3912,7 @@ namespace ranges { #if _USE_STD_VECTOR_ALGORITHMS if constexpr (is_same_v<_Pj, identity> && _Vector_alg_in_replace_is_safe<_It, _Ty1, _Ty2> && sized_sentinel_for<_Se, _It>) { - if (!_STD is_constant_evaluated()) - { + if (!_STD is_constant_evaluated()) { const auto _Count = _Last - _First; if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 20f8acab89c..b36d9de3f49 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -562,7 +562,7 @@ void test_case_replace(const vector& input, T old_val, T new_val) { template void test_replace(mt19937_64& gen) { - using TD = conditional_t; + using TD = conditional_t; uniform_int_distribution dis(0, 9); vector input; @@ -776,7 +776,7 @@ void test_vector_algorithms(mt19937_64& gen) { test_replace(gen); test_replace(gen); test_replace(gen); - + test_reverse(gen); test_reverse(gen); test_reverse(gen); From 79825dc09f9371734b353124f8879db627307af4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 15:36:08 +0300 Subject: [PATCH 03/18] intrinsic limitations --- stl/src/vector_algorithms.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 629f3ee52ae..12a2f52e183 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2359,8 +2359,15 @@ __declspec(noalias) void __stdcall __std_replace_trivial_4( __declspec(noalias) void __stdcall __std_replace_trivial_8( void* _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { if (_Use_avx2()) { +#ifdef _WIN64 const __m256i _Comparand = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_Old_val)); const __m256i _Replacement = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_New_val)); +#else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv + // Workaround, _mm_cvtsi64_si128 does not compile + const __m256i _Comparand = _mm256_set1_epi64x(_Old_val); + const __m256i _Replacement = _mm256_set1_epi64x(_New_val); +#endif // ^^^ !defined(_WIN64) ^^^ + const size_t _Full_length = _Byte_length(_First, _Last); From eb9175943e02e560e2ef818e27603e0c03c39b02 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 15:42:08 +0300 Subject: [PATCH 04/18] format --- stl/src/vector_algorithms.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 12a2f52e183..4bbc248149a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2362,8 +2362,7 @@ __declspec(noalias) void __stdcall __std_replace_trivial_8( #ifdef _WIN64 const __m256i _Comparand = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_Old_val)); const __m256i _Replacement = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_New_val)); -#else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv - // Workaround, _mm_cvtsi64_si128 does not compile +#else // ^^^ defined(_WIN64) / !defined(_WIN64), workaround, _mm_cvtsi64_si128 does not compile vvv const __m256i _Comparand = _mm256_set1_epi64x(_Old_val); const __m256i _Replacement = _mm256_set1_epi64x(_New_val); #endif // ^^^ !defined(_WIN64) ^^^ From 9b8eae58c372a02bba9d7b757fa496fef9103e60 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 19:18:20 +0300 Subject: [PATCH 05/18] Don't check replacement type if it is the same --- stl/inc/algorithm | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 8c0a6c30312..ef4f5e428ed 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -216,11 +216,17 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = && sizeof(_Iter_value_t<_It1>) <= 2; // pcmpestri compatible size // Can we activate the vector algorithms for replace? -template +template _INLINE_VAR constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value && sizeof(_Iter_value_t<_Iter>) >= 4 // avx masked op compatible size && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits + +// Can we activate the vector algorithms for replace? +template +_INLINE_VAR constexpr bool _Vector_alg_in_replace_with_maybe_other_type_is_safe = + _Vector_alg_in_replace_is_safe<_Iter, _Ty1> // can search and replace + && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -3842,7 +3848,7 @@ _CONSTEXPR20 void replace(const _FwdIt _First, const _FwdIt _Last, const _Ty& _O const auto _ULast = _STD _Get_unwrapped(_Last); #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (_Vector_alg_in_replace_is_safe<_FwdIt, _Ty, _Ty>) { + if constexpr (_Vector_alg_in_replace_is_safe<_FwdIt, _Ty>) { #if _HAS_CXX20 if (!_STD is_constant_evaluated()) #endif // _HAS_CXX20 @@ -3910,8 +3916,8 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(indirect_binary_predicate, const _Ty1*>); #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (is_same_v<_Pj, identity> && _Vector_alg_in_replace_is_safe<_It, _Ty1, _Ty2> - && sized_sentinel_for<_Se, _It>) { + if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> + && _Vector_alg_in_replace_with_maybe_other_type_is_safe<_It, _Ty1, _Ty2>) { if (!_STD is_constant_evaluated()) { const auto _Count = _Last - _First; From f903576595aad1fd0da3948053a518844aaa6acf Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 19:19:42 +0300 Subject: [PATCH 06/18] Don't check it here --- stl/inc/algorithm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index ef4f5e428ed..0fcb5ce0ed8 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -219,8 +219,7 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = template _INLINE_VAR constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value - && sizeof(_Iter_value_t<_Iter>) >= 4 // avx masked op compatible size - && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits + && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size // Can we activate the vector algorithms for replace? template From 35356b48ee9809fa1e3c2205c7a6d406958cf8f7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 19:37:44 +0300 Subject: [PATCH 07/18] constexpr is new inline --- stl/inc/algorithm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 0fcb5ce0ed8..4463b81284b 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -217,13 +217,13 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = // Can we activate the vector algorithms for replace? template -_INLINE_VAR constexpr bool _Vector_alg_in_replace_is_safe = +constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size // Can we activate the vector algorithms for replace? template -_INLINE_VAR constexpr bool _Vector_alg_in_replace_with_maybe_other_type_is_safe = +constexpr bool _Vector_alg_in_replace_with_maybe_other_type_is_safe = _Vector_alg_in_replace_is_safe<_Iter, _Ty1> // can search and replace && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits _STD_END From ba828cb417075b38d4442cc8077d72115b9cd707 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 19:39:17 +0300 Subject: [PATCH 08/18] less lines --- stl/src/vector_algorithms.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 4bbc248149a..646e56c7ec2 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2327,8 +2327,7 @@ __declspec(noalias) void __stdcall __std_replace_trivial_4( if (_Use_avx2()) { const __m256i _Comparand = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_Old_val)); const __m256i _Replacement = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_New_val)); - - const size_t _Full_length = _Byte_length(_First, _Last); + const size_t _Full_length = _Byte_length(_First, _Last); void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Full_length & ~size_t{0x1F}); @@ -2366,8 +2365,6 @@ __declspec(noalias) void __stdcall __std_replace_trivial_8( const __m256i _Comparand = _mm256_set1_epi64x(_Old_val); const __m256i _Replacement = _mm256_set1_epi64x(_New_val); #endif // ^^^ !defined(_WIN64) ^^^ - - const size_t _Full_length = _Byte_length(_First, _Last); void* _Stop_at = _First; From 318e40d1523cd0b162c5a0a17dcd8afd89295360 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 19:47:30 +0300 Subject: [PATCH 09/18] format --- stl/inc/algorithm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 4463b81284b..40a7c30cc1e 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -217,9 +217,8 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = // Can we activate the vector algorithms for replace? template -constexpr bool _Vector_alg_in_replace_is_safe = - _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value - && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size +constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value + && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size // Can we activate the vector algorithms for replace? template From c728328cb59768b9f9f46deff17a4c6ea68b5436 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 3 Apr 2024 22:02:48 +0300 Subject: [PATCH 10/18] Meow. Vectorized. --- stl/inc/algorithm | 20 +++++++++----------- stl/src/vector_algorithms.cpp | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 40a7c30cc1e..41f5c0040d7 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -75,9 +75,9 @@ __declspec(noalias) _Min_max_f __stdcall __std_minmax_f(const void* _First, cons __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* _First, const void* _Last) noexcept; // TRANSITION, DevCom-10610477 -__declspec(noalias) void __stdcall __std_replace_trivial_4( +__declspec(noalias) void __stdcall __std_replace_4( void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept; -__declspec(noalias) void __stdcall __std_replace_trivial_8( +__declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; } // extern "C" @@ -187,20 +187,18 @@ _Ty1* __std_find_first_of_trivial( } template -__declspec(noalias) void _Replace_trivial( +__declspec(noalias) void _Replace_vectorized( _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { if constexpr (is_pointer_v<_Ty>) { #ifdef _WIN64 - ::__std_replace_trivial_8( - _First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); + ::__std_replace_8(_First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); #else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv - ::__std_replace_trivial_4( - _First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); + ::__std_replace_4(_First, _Last, reinterpret_cast(_Old_val), reinterpret_cast(_New_val)); #endif // ^^^ !defined(_WIN64) ^^^ } else if constexpr (sizeof(_Ty) == 4) { - ::__std_replace_trivial_4(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); + ::__std_replace_4(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); } else if constexpr (sizeof(_Ty) == 8) { - ::__std_replace_trivial_8(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); + ::__std_replace_8(_First, _Last, static_cast(_Old_val), static_cast(_New_val)); } else { static_assert(_Always_false<_Ty>, "Unexpected size"); } @@ -3852,7 +3850,7 @@ _CONSTEXPR20 void replace(const _FwdIt _First, const _FwdIt _Last, const _Ty& _O #endif // _HAS_CXX20 { if (_STD _Could_compare_equal_to_value_type<_FwdIt>(_Oldval)) { - _STD _Replace_trivial(_STD _To_address(_UFirst), _STD _To_address(_ULast), _Oldval, _Newval); + _STD _Replace_vectorized(_STD _To_address(_UFirst), _STD _To_address(_ULast), _Oldval, _Newval); } return; @@ -3922,7 +3920,7 @@ namespace ranges { if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { const auto _First_ptr = _STD to_address(_First); const auto _Last_ptr = _First_ptr + _Count; - _STD _Replace_trivial(_First_ptr, _Last_ptr, _Oldval, _Newval); + _STD _Replace_vectorized(_First_ptr, _Last_ptr, _Oldval, _Newval); } return _First + _Count; diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 646e56c7ec2..b6aff46ec6c 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2322,7 +2322,7 @@ namespace { extern "C" { -__declspec(noalias) void __stdcall __std_replace_trivial_4( +__declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { if (_Use_avx2()) { const __m256i _Comparand = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_Old_val)); @@ -2355,7 +2355,7 @@ __declspec(noalias) void __stdcall __std_replace_trivial_4( } } -__declspec(noalias) void __stdcall __std_replace_trivial_8( +__declspec(noalias) void __stdcall __std_replace_8( void* _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { if (_Use_avx2()) { #ifdef _WIN64 From 5407aebe15f32b98b2041b9b8216fb9a70cd6434 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 09:06:07 -0700 Subject: [PATCH 11/18] Move `__std_replace_MEOW` up, still within `extern "C"`, to avoid splitting the bitset machinery. No other changes. --- stl/src/vector_algorithms.cpp | 100 +++++++++++++++++----------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b6aff46ec6c..9d53e29169d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2272,56 +2272,6 @@ __declspec(noalias) size_t return __std_mismatch_impl<_Find_traits_8, uint64_t>(_First1, _First2, _Count); } -} // extern "C" - -#ifndef _M_ARM64EC -namespace { - __m256i __forceinline _Bitset_to_string_1_step_avx(const uint32_t _Val, const __m256i _Px0, const __m256i _Px1) { - const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); - const __m128i _Vx1 = _mm_shuffle_epi8(_Vx0, _mm_set_epi32(0x00000000, 0x01010101, 0x02020202, 0x03030303)); - const __m256i _Vx2 = _mm256_castsi128_si256(_Vx1); - const __m256i _Vx3 = _mm256_permutevar8x32_epi32(_Vx2, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); - const __m256i _Msk = _mm256_and_si256(_Vx3, _mm256_set1_epi64x(0x0102040810204080)); - const __m256i _Ex0 = _mm256_cmpeq_epi8(_Msk, _mm256_setzero_si256()); - const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); - return _Ex1; - } - - __m128i __forceinline _Bitset_to_string_1_step(const uint16_t _Val, const __m128i _Px0, const __m128i _Px1) { - const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); - const __m128i _Vx1 = _mm_unpacklo_epi8(_Vx0, _Vx0); - const __m128i _Vx2 = _mm_unpacklo_epi8(_Vx1, _Vx1); - const __m128i _Vx3 = _mm_shuffle_epi32(_Vx2, _MM_SHUFFLE(0, 0, 1, 1)); - const __m128i _Msk = _mm_and_si128(_Vx3, _mm_set1_epi64x(0x0102040810204080)); - const __m128i _Ex0 = _mm_cmpeq_epi8(_Msk, _mm_setzero_si128()); - const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1); - return _Ex1; - } - - __m256i __forceinline _Bitset_to_string_2_step_avx(const uint16_t _Val, const __m256i _Px0, const __m256i _Px1) { - const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); - const __m128i _Vx1 = _mm_shuffle_epi8(_Vx0, _mm_set_epi32(0x00000000, 0x00000000, 0x01010101, 0x01010101)); - const __m256i _Vx2 = _mm256_castsi128_si256(_Vx1); - const __m256i _Vx3 = _mm256_permute4x64_epi64(_Vx2, _MM_SHUFFLE(1, 1, 0, 0)); - const __m256i _Msk = _mm256_and_si256( - _Vx3, _mm256_set_epi64x(0x0001000200040008, 0x0010002000400080, 0x0001000200040008, 0x0010002000400080)); - const __m256i _Ex0 = _mm256_cmpeq_epi16(_Msk, _mm256_setzero_si256()); - const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); - return _Ex1; - } - - __m128i __forceinline _Bitset_to_string_2_step(const uint8_t _Val, const __m128i _Px0, const __m128i _Px1) { - const __m128i _Vx = _mm_set1_epi16(_Val); - const __m128i _Msk = _mm_and_si128(_Vx, _mm_set_epi64x(0x0001000200040008, 0x0010002000400080)); - const __m128i _Ex0 = _mm_cmpeq_epi16(_Msk, _mm_setzero_si128()); - const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1); - return _Ex1; - } -} // unnamed namespace -#endif // !defined(_M_ARM64EC) - -extern "C" { - __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { if (_Use_avx2()) { @@ -2393,6 +2343,56 @@ __declspec(noalias) void __stdcall __std_replace_8( } } +} // extern "C" + +#ifndef _M_ARM64EC +namespace { + __m256i __forceinline _Bitset_to_string_1_step_avx(const uint32_t _Val, const __m256i _Px0, const __m256i _Px1) { + const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); + const __m128i _Vx1 = _mm_shuffle_epi8(_Vx0, _mm_set_epi32(0x00000000, 0x01010101, 0x02020202, 0x03030303)); + const __m256i _Vx2 = _mm256_castsi128_si256(_Vx1); + const __m256i _Vx3 = _mm256_permutevar8x32_epi32(_Vx2, _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0)); + const __m256i _Msk = _mm256_and_si256(_Vx3, _mm256_set1_epi64x(0x0102040810204080)); + const __m256i _Ex0 = _mm256_cmpeq_epi8(_Msk, _mm256_setzero_si256()); + const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); + return _Ex1; + } + + __m128i __forceinline _Bitset_to_string_1_step(const uint16_t _Val, const __m128i _Px0, const __m128i _Px1) { + const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); + const __m128i _Vx1 = _mm_unpacklo_epi8(_Vx0, _Vx0); + const __m128i _Vx2 = _mm_unpacklo_epi8(_Vx1, _Vx1); + const __m128i _Vx3 = _mm_shuffle_epi32(_Vx2, _MM_SHUFFLE(0, 0, 1, 1)); + const __m128i _Msk = _mm_and_si128(_Vx3, _mm_set1_epi64x(0x0102040810204080)); + const __m128i _Ex0 = _mm_cmpeq_epi8(_Msk, _mm_setzero_si128()); + const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1); + return _Ex1; + } + + __m256i __forceinline _Bitset_to_string_2_step_avx(const uint16_t _Val, const __m256i _Px0, const __m256i _Px1) { + const __m128i _Vx0 = _mm_cvtsi32_si128(_Val); + const __m128i _Vx1 = _mm_shuffle_epi8(_Vx0, _mm_set_epi32(0x00000000, 0x00000000, 0x01010101, 0x01010101)); + const __m256i _Vx2 = _mm256_castsi128_si256(_Vx1); + const __m256i _Vx3 = _mm256_permute4x64_epi64(_Vx2, _MM_SHUFFLE(1, 1, 0, 0)); + const __m256i _Msk = _mm256_and_si256( + _Vx3, _mm256_set_epi64x(0x0001000200040008, 0x0010002000400080, 0x0001000200040008, 0x0010002000400080)); + const __m256i _Ex0 = _mm256_cmpeq_epi16(_Msk, _mm256_setzero_si256()); + const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); + return _Ex1; + } + + __m128i __forceinline _Bitset_to_string_2_step(const uint8_t _Val, const __m128i _Px0, const __m128i _Px1) { + const __m128i _Vx = _mm_set1_epi16(_Val); + const __m128i _Msk = _mm_and_si128(_Vx, _mm_set_epi64x(0x0001000200040008, 0x0010002000400080)); + const __m128i _Ex0 = _mm_cmpeq_epi16(_Msk, _mm_setzero_si128()); + const __m128i _Ex1 = _mm_xor_si128(_mm_and_si128(_Ex0, _Px0), _Px1); + return _Ex1; + } +} // unnamed namespace +#endif // !defined(_M_ARM64EC) + +extern "C" { + __declspec(noalias) void __stdcall __std_bitset_to_string_1( char* const _Dest, const void* _Src, size_t _Size_bits, const char _Elem0, const char _Elem1) noexcept { #ifndef _M_ARM64EC From 9a6a0d541525bcc06d6be8d480b686219baba097 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 09:18:04 -0700 Subject: [PATCH 12/18] Add `#ifndef _M_ARM64EC` guards. --- stl/src/vector_algorithms.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 9d53e29169d..d4600505aa1 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2274,6 +2274,7 @@ __declspec(noalias) size_t __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { +#ifndef _M_ARM64EC if (_Use_avx2()) { const __m256i _Comparand = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_Old_val)); const __m256i _Replacement = _mm256_broadcastd_epi32(_mm_cvtsi32_si128(_New_val)); @@ -2296,7 +2297,9 @@ __declspec(noalias) void __stdcall __std_replace_4( const __m256i _Mask = _mm256_and_si256(_mm256_cmpeq_epi32(_Comparand, _Data), _Tail_mask); _mm256_maskstore_epi32(reinterpret_cast(_First), _Mask, _Replacement); } - } else { + } else +#endif // !defined(_M_ARM64EC) + { for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { if (*_Cur == _Old_val) { *_Cur = _New_val; @@ -2307,6 +2310,7 @@ __declspec(noalias) void __stdcall __std_replace_4( __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* const _Last, const uint64_t _Old_val, const uint64_t _New_val) noexcept { +#ifndef _M_ARM64EC if (_Use_avx2()) { #ifdef _WIN64 const __m256i _Comparand = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(_Old_val)); @@ -2334,7 +2338,9 @@ __declspec(noalias) void __stdcall __std_replace_8( const __m256i _Mask = _mm256_and_si256(_mm256_cmpeq_epi64(_Comparand, _Data), _Tail_mask); _mm256_maskstore_epi64(reinterpret_cast(_First), _Mask, _Replacement); } - } else { + } else +#endif // !defined(_M_ARM64EC) + { for (auto _Cur = reinterpret_cast(_First); _Cur != _Last; ++_Cur) { if (*_Cur == _Old_val) { *_Cur = _New_val; From 4e121a037327643db1b1780412970e22c552ab65 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 14:27:48 -0700 Subject: [PATCH 13/18] Generate one value per line. --- .../std/tests/VSO_0000000_vector_algorithms/test.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index b36d9de3f49..7ade1b439a3 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -568,10 +568,17 @@ void test_replace(mt19937_64& gen) { input.reserve(dataCount); - test_case_replace(input, static_cast(dis(gen)), static_cast(dis(gen))); + { + const T old_val = static_cast(dis(gen)); + const T new_val = static_cast(dis(gen)); + test_case_replace(input, old_val, new_val); + } + for (size_t i = 0; i != dataCount; ++i) { input.push_back(static_cast(dis(gen))); - test_case_replace(input, static_cast(dis(gen)), static_cast(dis(gen))); + const T old_val = static_cast(dis(gen)); + const T new_val = static_cast(dis(gen)); + test_case_replace(input, old_val, new_val); } } From ec19ed9a814d75280acc08a4b83cbe825171aeba Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 14:47:29 -0700 Subject: [PATCH 14/18] Test only 4 and 8 bytes, add comments. --- benchmarks/src/replace.cpp | 1 + tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index 4c678229575..fe5bc68641a 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -69,6 +69,7 @@ void rc_if(benchmark::State& state) { } } +// replace() is vectorized for 4 and 8 bytes only. BENCHMARK(r); BENCHMARK(r); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 7ade1b439a3..39eed458ad3 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -774,11 +774,7 @@ void test_vector_algorithms(mt19937_64& gen) { test_mismatch_sizes_and_alignments::test(); test_mismatch_sizes_and_alignments::test(); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); + // replace() is vectorized for 4 and 8 bytes only. test_replace(gen); test_replace(gen); test_replace(gen); From b761c6761626de25c4a5c79a7cebfa1d79c9b58e Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 15:18:00 -0700 Subject: [PATCH 15/18] Rename to `_Vector_alg_in_ranges_replace_is_safe`. --- stl/inc/algorithm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 41f5c0040d7..baab88d09e5 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -218,9 +218,9 @@ template constexpr bool _Vector_alg_in_replace_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty1> // can search for the value && sizeof(_Iter_value_t<_Iter>) >= 4; // avx masked op compatible size -// Can we activate the vector algorithms for replace? +// Can we activate the vector algorithms for ranges::replace? template -constexpr bool _Vector_alg_in_replace_with_maybe_other_type_is_safe = +constexpr bool _Vector_alg_in_ranges_replace_is_safe = _Vector_alg_in_replace_is_safe<_Iter, _Ty1> // can search and replace && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits _STD_END @@ -3913,7 +3913,7 @@ namespace ranges { #if _USE_STD_VECTOR_ALGORITHMS if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> - && _Vector_alg_in_replace_with_maybe_other_type_is_safe<_It, _Ty1, _Ty2>) { + && _Vector_alg_in_ranges_replace_is_safe<_It, _Ty1, _Ty2>) { if (!_STD is_constant_evaluated()) { const auto _Count = _Last - _First; From 3bb6c62049038851f9dc28534a4b1ba84795653e Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 15:22:23 -0700 Subject: [PATCH 16/18] Inspect the unwrapped iterator in classic replace(). --- stl/inc/algorithm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index baab88d09e5..a82ca081ba2 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -3844,12 +3844,12 @@ _CONSTEXPR20 void replace(const _FwdIt _First, const _FwdIt _Last, const _Ty& _O const auto _ULast = _STD _Get_unwrapped(_Last); #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (_Vector_alg_in_replace_is_safe<_FwdIt, _Ty>) { + if constexpr (_Vector_alg_in_replace_is_safe) { #if _HAS_CXX20 if (!_STD is_constant_evaluated()) #endif // _HAS_CXX20 { - if (_STD _Could_compare_equal_to_value_type<_FwdIt>(_Oldval)) { + if (_STD _Could_compare_equal_to_value_type(_Oldval)) { _STD _Replace_vectorized(_STD _To_address(_UFirst), _STD _To_address(_ULast), _Oldval, _Newval); } From d3c95d06634e003a100e2d4fd90b0c2eb210eec3 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 15:37:01 -0700 Subject: [PATCH 17/18] Drop `_INLINE_VAR` to mitigate conflicts with GH 4546. --- stl/inc/xutility | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index b91965d19bd..c1ae7484e76 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -5832,7 +5832,7 @@ struct _Vector_alg_in_find_is_safe_object_pointers<_Ty1*, _Ty2*> // Can we activate the vector algorithms for a value and container elements template -_INLINE_VAR constexpr bool _Vector_alg_in_find_is_safe_elem = disjunction_v< +constexpr bool _Vector_alg_in_find_is_safe_elem = disjunction_v< #ifdef __cpp_lib_byte // We're finding a std::byte in a range of std::byte. conjunction, is_same<_Elem, byte>>, @@ -5851,7 +5851,7 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_is_safe_elem = disjunction_v< // Can we activate the vector algorithms for find/count? template -_INLINE_VAR constexpr bool _Vector_alg_in_find_is_safe = +constexpr bool _Vector_alg_in_find_is_safe = // The iterator must be contiguous so we can get raw pointers. _Iterator_is_contiguous<_Iter> // The iterator must not be volatile. From 120e010784e19bcd7784e275e35f2ab5b33aa0c8 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 4 Apr 2024 16:04:06 -0700 Subject: [PATCH 18/18] Clarify comments. --- stl/inc/xutility | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index c1ae7484e76..61e22bc933b 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -5830,7 +5830,7 @@ struct _Vector_alg_in_find_is_safe_object_pointers<_Ty1*, _Ty2*> // either _Ty1 is the same as _Ty2 (ignoring cv-qualifiers), or one of the two is void disjunction, remove_cv_t<_Ty2>>, is_void<_Ty1>, is_void<_Ty2>>> {}; -// Can we activate the vector algorithms for a value and container elements +// Can we activate the vector algorithms to find a value in a range of elements? template constexpr bool _Vector_alg_in_find_is_safe_elem = disjunction_v< #ifdef __cpp_lib_byte @@ -5856,7 +5856,7 @@ constexpr bool _Vector_alg_in_find_is_safe = _Iterator_is_contiguous<_Iter> // The iterator must not be volatile. && !_Iterator_is_volatile<_Iter> - // The elements and the value of a certain matching types. + // The type of the value to find must be compatible with the type of the elements. && _Vector_alg_in_find_is_safe_elem<_Ty, _Iter_value_t<_Iter>>; template