diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index ebb1aa038bf..28ab1c12ddb 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -34,19 +34,6 @@ void rc(benchmark::State& state) { } } -template -void rc_if(benchmark::State& state) { - std::vector> a(lorem_ipsum.begin(), lorem_ipsum.end()); - std::vector> b(lorem_ipsum.size()); - - for (auto _ : state) { - benchmark::DoNotOptimize(a); - (void) std::replace_copy_if( - std::begin(a), std::end(a), std::begin(b), [](auto x) { return x <= T{'Z'}; }, T{'X'}); - benchmark::DoNotOptimize(b); - } -} - // replace() is vectorized for 4 and 8 bytes only. BENCHMARK(r); BENCHMARK(r); @@ -56,9 +43,4 @@ BENCHMARK(rc); BENCHMARK(rc); BENCHMARK(rc); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); - BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 3fc1b239309..4d7f0a23852 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -122,6 +122,17 @@ __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +__declspec(noalias) void __stdcall __std_replace_copy_1( + const void* _First, const void* _Last, void* _Dest, uint8_t _Old_val, uint8_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_2( + const void* _First, const void* _Last, void* _Dest, uint16_t _Old_val, uint16_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_4( + const void* _First, const void* _Last, void* _Dest, uint32_t _Old_val, uint32_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_8( + const void* _First, const void* _Last, void* _Dest, uint64_t _Old_val, uint64_t _New_val) noexcept; +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N const void* __stdcall __std_search_n_1(const void* _First, const void* _Last, size_t _Count, uint8_t _Value) noexcept; const void* __stdcall __std_search_n_2(const void* _First, const void* _Last, size_t _Count, uint16_t _Value) noexcept; @@ -358,6 +369,28 @@ __declspec(noalias) void _Replace_vectorized( } #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +template +__declspec(noalias) void _Replace_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _Ty* const _Dest, + const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { + if constexpr (sizeof(_Ty) == 1) { + ::__std_replace_copy_1( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 2) { + ::__std_replace_copy_2( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 4) { + ::__std_replace_copy_4( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 8) { + ::__std_replace_copy_8( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else { + static_assert(false, "unexpected size"); + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N template _Ty* _Search_n_vectorized(_Ty* const _First, _Ty* const _Last, const size_t _Count, const _TVal _Val) noexcept { @@ -476,6 +509,17 @@ constexpr bool _Output_iterator_for_vector_alg_is_safe() { } #endif // ^^^ _VECTORIZED_REMOVE_COPY || _VECTORIZED_UNIQUE_COPY ^^^ +#if _VECTORIZED_REPLACE_COPY +template +constexpr bool _Output_iterator_for_known_size_vector_alg_is_safe() { + if constexpr (_Iterator_is_contiguous<_Out>) { + return is_same_v<_Iter_value_t<_Out>, remove_const_t<_Iter_value_t<_In>>>; + } else { + return false; + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_INCLUDES // Can we activate the vector algorithms for includes? template > @@ -4562,15 +4606,6 @@ namespace ranges { } // namespace ranges #endif // _HAS_CXX20 -// TRANSITION, DevCom-10606350: help the compiler auto-vectorize for simple types -template > -constexpr bool _Can_vectorize_replace_copy = conjunction_v, is_same<_InTy, _NewTy>, - disjunction< -#ifdef __cpp_lib_byte - conjunction, is_same<_OutTy, byte>>, -#endif // defined(__cpp_lib_byte) - conjunction, is_integral<_OutTy>>, conjunction, is_pointer<_OutTy>>>>; - _EXPORT_STD template _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _Ty& _Oldval, const _Ty& _Newval) { // copy replacing each matching _Oldval with _Newval @@ -4579,15 +4614,36 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); - for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = *_UFirst == _Oldval ? _Newval : *_UFirst; - } else { - if (*_UFirst == _Oldval) { - *_UDest = _Newval; + +#if _VECTORIZED_REPLACE_COPY + if constexpr (_Vector_alg_in_find_is_safe + && _Output_iterator_for_known_size_vector_alg_is_safe()) { + if (!_STD _Is_constant_evaluated()) { + const auto _Count = static_cast<_Iter_diff_t>(_ULast - _UFirst); + _STD _Contiguous_iter_verify(_UDest, _Count); + + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Last_ptr = _STD _To_address(_ULast); + const auto _Dest_ptr = _STD _To_address(_UDest); + + if (_STD _Could_compare_equal_to_value_type(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Dest_ptr, _Oldval, _Newval); } else { - *_UDest = *_UFirst; + _CSTD memcpy(_Dest_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Dest_ptr)); } + + _UDest += _Count; + _STD _Seek_wrapped(_Dest, _UDest); + return _Dest; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + + for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { + if (*_UFirst == _Oldval) { + *_UDest = _Newval; + } else { + *_UDest = *_UFirst; } } @@ -4660,15 +4716,37 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); - for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty2>) { - *_Output = _STD invoke(_Proj, *_First) == _Oldval ? _Newval : *_First; - } else { - if (_STD invoke(_Proj, *_First) == _Oldval) { - *_Output = _Newval; +#if _VECTORIZED_REPLACE_COPY + if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> + && _Vector_alg_in_find_is_safe<_It, _Ty1> && _Vector_alg_in_find_is_safe<_It, _Ty2> + && _Output_iterator_for_known_size_vector_alg_is_safe<_Out, _It>()) { + if (!_STD is_constant_evaluated()) { + const auto _Count = _Last - _First; + _STD _Contiguous_iter_verify(_First, _Count); + _STD _Contiguous_iter_verify(_Output, static_cast>(_Count)); + + const auto _First_ptr = _STD to_address(_First); + const auto _Last_ptr = _First_ptr + static_cast(_Count); + const auto _Out_ptr = _STD to_address(_Output); + + if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Out_ptr, _Oldval, _Newval); } else { - *_Output = *_First; + _CSTD memcpy(_Out_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Out_ptr)); } + + _First += _Count; + _Output += static_cast>(_Count); + return {_STD move(_First), _STD move(_Output)}; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + + for (; _First != _Last; ++_First, (void) ++_Output) { + if (_STD invoke(_Proj, *_First) == _Oldval) { + *_Output = _Newval; + } else { + *_Output = *_First; } } @@ -4689,14 +4767,10 @@ _CONSTEXPR20 _OutIt replace_copy_if(_InIt _First, _InIt _Last, _OutIt _Dest, _Pr const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = _Pred(*_UFirst) ? _Val : *_UFirst; + if (_Pred(*_UFirst)) { + *_UDest = _Val; } else { - if (_Pred(*_UFirst)) { - *_UDest = _Val; - } else { - *_UDest = *_UFirst; - } + *_UDest = *_UFirst; } } @@ -4771,14 +4845,10 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty>) { - *_Output = _STD invoke(_Pred, _STD invoke(_Proj, *_First)) ? _Newval : *_First; + if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { + *_Output = _Newval; } else { - if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { - *_Output = _Newval; - } else { - *_Output = *_First; - } + *_Output = *_First; } } diff --git a/stl/inc/xutility b/stl/inc/xutility index 53a64c6c787..f14bd773894 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -95,6 +95,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REPLACE _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_REPLACE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REVERSE _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_REVERSE_COPY _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86_ARM64 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cb8bbbdbca0..ba40ad57138 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6486,6 +6486,80 @@ __declspec(noalias) size_t __stdcall __std_mismatch_8( return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } +} // extern "C" + +namespace { + namespace _Replacing { + template + __declspec(noalias) void __stdcall _Replace_copy_impl( + const void* _First, const void* const _Last, void* _Dest, const _Ty _Old_val, const _Ty _New_val) noexcept { +#ifndef _M_ARM64EC + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { + const __m256i _Comparand = _Traits::_Set_avx(_Old_val); + const __m256i _Replacement = _Traits::_Set_avx(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Avx_size); + + do { + const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Val); + + _Advance_bytes(_First, 32); + _Advance_bytes(_Dest, 32); + } while (_First != _Stop_at); + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_maskstore_epi32(static_cast(_Dest), _Tail_mask, _Val); + + _Advance_bytes(_First, _Avx_tail_size); + _Advance_bytes(_Dest, _Avx_tail_size); + } + + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { + const __m128i _Comparand = _Traits::_Set_sse(_Old_val); + const __m128i _Replacement = _Traits::_Set_sse(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Sse_size); + + do { + const __m128i _Data = _mm_loadu_si128(static_cast(_First)); + const __m128i _Mask = _Traits::_Cmp_sse(_Data, _Comparand); + const __m128i _Val = _mm_blendv_epi8(_Data, _Replacement, _Mask); + + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Val); + + _Advance_bytes(_First, 16); + _Advance_bytes(_Dest, 16); + } while (_First != _Stop_at); + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + auto _Ptr_dest = static_cast<_Ty*>(_Dest); + for (auto _Ptr_src = static_cast(_First); _Ptr_src != _Last; ++_Ptr_src) { + const _Ty _Val = *_Ptr_src; + *_Ptr_dest = _Val == _Old_val ? _New_val : _Val; + ++_Ptr_dest; + } + } + } // namespace _Replacing +} // unnamed namespace + +extern "C" { + __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { #ifndef _M_ARM64EC @@ -6567,6 +6641,26 @@ __declspec(noalias) void __stdcall __std_replace_8( } } +__declspec(noalias) void __stdcall __std_replace_copy_1(const void* const _First, const void* const _Last, + void* const _Dest, const uint8_t _Old_val, const uint8_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_1>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_2(const void* const _First, const void* const _Last, + void* const _Dest, const uint16_t _Old_val, const uint16_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_2>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_4(const void* const _First, const void* const _Last, + void* const _Dest, const uint32_t _Old_val, const uint32_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_4>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_8(const void* const _First, const void* const _Last, + void* const _Dest, const uint64_t _Old_val, const uint64_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_8>(_First, _Last, _Dest, _Old_val, _New_val); +} + } // extern "C" namespace { diff --git a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp index c2b53e59f58..0a82d9bad27 100644 --- a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp +++ b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp @@ -221,11 +221,17 @@ int main() { assert(r_rot_it == temp_end - rotate_pos); } { - // Out of replace family, only replace for 32-bit and 64-bit elements is manually vectorized, - // replace_copy is auto vectorized (along with replace_copy_if) const int replace_expected[] = { 200, 210, 220, 333, 240, 333, 333, 270, 280, 290, 300, 310, 320, 333, 340, 333, 333, 370, 380, 390}; + auto repl_copy_it = replace_copy(arr_begin, arr_end, temp_begin, 250, 333); + assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(repl_copy_it == temp_end); + + auto r_repl_copy_it = ranges::replace_copy(arr_begin, arr_end, temp_begin, 250, 333).out; + assert(ranges::equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(r_repl_copy_it == temp_end); + copy(arr_begin, arr_end, temp_begin); replace(temp_begin, temp_end, 250, 333); assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index eeff29a4bfa..3a77c979cf9 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -723,40 +723,101 @@ void last_known_good_replace(FwdIt first, FwdIt last, const T old_val, const T n } } +template +void last_known_good_replace_copy(FwdIt first, FwdIt last, OutIt dest, const T old_val, const T new_val) { + for (; first != last; ++first, ++dest) { + if (*first == old_val) { + *dest = new_val; + } else { + *dest = *first; + } + } +} + template -void test_case_replace(const vector& input, T old_val, T new_val) { - vector replaced_actual(input); - vector replaced_expected(input); - replace(replaced_actual.begin(), replaced_actual.end(), old_val, new_val); - last_known_good_replace(replaced_expected.begin(), replaced_expected.end(), old_val, new_val); - assert(replaced_expected == replaced_actual); +void test_case_replace(vector& in_out_expected, vector& in_out_actual, vector& in_out_actual_r, + const T old_val, const T new_val) { + replace(in_out_actual.begin(), in_out_actual.end(), old_val, new_val); + last_known_good_replace(in_out_expected.begin(), in_out_expected.end(), old_val, new_val); + assert(in_out_expected == in_out_actual); #if _HAS_CXX20 - vector replaced_actual_r(input); - ranges::replace(replaced_actual_r, old_val, new_val); - assert(replaced_expected == replaced_actual_r); -#endif // _HAS_CXX20 + ranges::replace(in_out_actual_r, old_val, new_val); + assert(in_out_expected == in_out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) in_out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ +} + +template +void test_case_replace_copy(const vector& input, vector& out_expected, vector& out_actual, + vector& out_actual_r, const T old_val, const T new_val) { + + replace_copy(input.begin(), input.end(), out_actual.begin(), old_val, new_val); + last_known_good_replace_copy(input.begin(), input.end(), out_expected.begin(), old_val, new_val); + assert(out_expected == out_actual); + +#if _HAS_CXX20 + ranges::replace_copy(input, out_actual_r.begin(), old_val, new_val); + assert(out_expected == out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ } template void test_replace(mt19937_64& gen) { + // replace() is vectorized for 4 and 8 bytes only. + constexpr bool replace_is_vectorized = sizeof(T) >= 4; + using TD = conditional_t; uniform_int_distribution dis(0, 9); - vector input; - input.reserve(dataCount); + vector source; + vector out_expected; + vector out_actual; + vector out_actual_r; + vector in_out_expected; + vector in_out_actual; + vector in_out_actual_r; + + for (const auto& v : {&source, &out_expected, &out_actual, &out_actual_r}) { + v->reserve(dataCount); + } + + if constexpr (replace_is_vectorized) { + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + v->reserve(dataCount); + } + } { const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + if constexpr (replace_is_vectorized) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } - for (size_t i = 0; i != dataCount; ++i) { - input.push_back(static_cast(dis(gen))); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + source.push_back(static_cast(dis(gen))); const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + *v = source; + } + + for (const auto& v : {&out_expected, &out_actual, &out_actual_r}) { + v->assign(source.size(), T{0}); + } + + if constexpr (replace_is_vectorized) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } } @@ -1287,7 +1348,11 @@ void test_vector_algorithms(mt19937_64& gen) { test_includes(gen); #endif // _HAS_CXX17 - // replace() is vectorized for 4 and 8 bytes only. + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); test_replace(gen); test_replace(gen); test_replace(gen);